@streamr/dht 100.0.0-testnet-three.6 → 100.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +1 -1
  2. package/dist/package.json +12 -8
  3. package/dist/src/connection/ConnectionLockHandler.d.ts +2 -2
  4. package/dist/src/connection/ConnectionLockHandler.js +13 -5
  5. package/dist/src/connection/ConnectionLockHandler.js.map +1 -1
  6. package/dist/src/connection/ConnectionManager.d.ts +4 -4
  7. package/dist/src/connection/ConnectionManager.js +8 -7
  8. package/dist/src/connection/ConnectionManager.js.map +1 -1
  9. package/dist/src/connection/ConnectorFacade.js.map +1 -1
  10. package/dist/src/connection/Handshaker.js +6 -13
  11. package/dist/src/connection/Handshaker.js.map +1 -1
  12. package/dist/src/connection/connectivityChecker.d.ts +1 -1
  13. package/dist/src/connection/connectivityChecker.js +5 -5
  14. package/dist/src/connection/connectivityChecker.js.map +1 -1
  15. package/dist/src/connection/connectivityRequestHandler.d.ts +2 -2
  16. package/dist/src/connection/connectivityRequestHandler.js +6 -7
  17. package/dist/src/connection/connectivityRequestHandler.js.map +1 -1
  18. package/dist/src/connection/webrtc/WebrtcConnector.d.ts +1 -0
  19. package/dist/src/connection/webrtc/WebrtcConnector.js +13 -8
  20. package/dist/src/connection/webrtc/WebrtcConnector.js.map +1 -1
  21. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.d.ts +2 -0
  22. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js +4 -6
  23. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js.map +1 -1
  24. package/dist/src/connection/websocket/WebsocketConnector.js +24 -21
  25. package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
  26. package/dist/src/connection/websocket/WebsocketServer.js +25 -35
  27. package/dist/src/connection/websocket/WebsocketServer.js.map +1 -1
  28. package/dist/src/connection/websocket/{ServerWebsocket.d.ts → WebsocketServerConnection.d.ts} +4 -5
  29. package/dist/src/connection/websocket/{ServerWebsocket.js → WebsocketServerConnection.js} +15 -48
  30. package/dist/src/connection/websocket/WebsocketServerConnection.js.map +1 -0
  31. package/dist/src/dht/DhtNode.d.ts +13 -6
  32. package/dist/src/dht/DhtNode.js +50 -15
  33. package/dist/src/dht/DhtNode.js.map +1 -1
  34. package/dist/src/dht/DhtNodeRpcLocal.d.ts +5 -1
  35. package/dist/src/dht/DhtNodeRpcLocal.js +10 -0
  36. package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
  37. package/dist/src/dht/DhtNodeRpcRemote.d.ts +3 -0
  38. package/dist/src/dht/DhtNodeRpcRemote.js +15 -0
  39. package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
  40. package/dist/src/dht/ExternalApiRpcLocal.d.ts +2 -2
  41. package/dist/src/dht/ExternalApiRpcLocal.js +3 -3
  42. package/dist/src/dht/ExternalApiRpcLocal.js.map +1 -1
  43. package/dist/src/dht/ExternalApiRpcRemote.d.ts +1 -1
  44. package/dist/src/dht/ExternalApiRpcRemote.js +2 -2
  45. package/dist/src/dht/ExternalApiRpcRemote.js.map +1 -1
  46. package/dist/src/dht/PeerManager.d.ts +12 -0
  47. package/dist/src/dht/PeerManager.js +30 -5
  48. package/dist/src/dht/PeerManager.js.map +1 -1
  49. package/dist/src/dht/contact/RingContactList.d.ts +31 -0
  50. package/dist/src/dht/contact/RingContactList.js +133 -0
  51. package/dist/src/dht/contact/RingContactList.js.map +1 -0
  52. package/dist/src/dht/contact/ringIdentifiers.d.ts +16 -0
  53. package/dist/src/dht/contact/ringIdentifiers.js +54 -0
  54. package/dist/src/dht/contact/ringIdentifiers.js.map +1 -0
  55. package/dist/src/dht/discovery/PeerDiscovery.d.ts +4 -0
  56. package/dist/src/dht/discovery/PeerDiscovery.js +35 -0
  57. package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
  58. package/dist/src/dht/discovery/RingDiscoverySession.d.ts +29 -0
  59. package/dist/src/dht/discovery/RingDiscoverySession.js +123 -0
  60. package/dist/src/dht/discovery/RingDiscoverySession.js.map +1 -0
  61. package/dist/src/dht/recursive-operation/RecursiveOperationSession.js +0 -1
  62. package/dist/src/dht/recursive-operation/RecursiveOperationSession.js.map +1 -1
  63. package/dist/src/dht/store/StoreManager.js +1 -1
  64. package/dist/src/dht/store/StoreManager.js.map +1 -1
  65. package/dist/src/exports.d.ts +1 -0
  66. package/dist/src/exports.js.map +1 -1
  67. package/dist/src/helpers/createPeerDescriptor.d.ts +1 -1
  68. package/dist/src/helpers/createPeerDescriptor.js +2 -1
  69. package/dist/src/helpers/createPeerDescriptor.js.map +1 -1
  70. package/dist/src/helpers/version.d.ts +6 -0
  71. package/dist/src/helpers/version.js +38 -0
  72. package/dist/src/helpers/version.js.map +1 -0
  73. package/dist/src/proto/packages/dht/protos/DhtRpc.client.d.ts +16 -6
  74. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js +11 -4
  75. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js.map +1 -1
  76. package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +98 -87
  77. package/dist/src/proto/packages/dht/protos/DhtRpc.js +45 -49
  78. package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
  79. package/dist/src/proto/packages/dht/protos/DhtRpc.server.d.ts +10 -4
  80. package/dist/src/transport/RoutingRpcCommunicator.js +0 -2
  81. package/dist/src/transport/RoutingRpcCommunicator.js.map +1 -1
  82. package/package.json +12 -8
  83. package/protos/DhtRpc.proto +21 -21
  84. package/src/connection/ConnectionLockHandler.ts +13 -5
  85. package/src/connection/ConnectionManager.ts +10 -10
  86. package/src/connection/ConnectorFacade.ts +0 -1
  87. package/src/connection/Handshaker.ts +7 -15
  88. package/src/connection/connectivityChecker.ts +6 -7
  89. package/src/connection/connectivityRequestHandler.ts +12 -12
  90. package/src/connection/webrtc/NodeWebrtcConnection.ts +1 -1
  91. package/src/connection/webrtc/WebrtcConnector.ts +14 -8
  92. package/src/connection/webrtc/WebrtcConnectorRpcLocal.ts +5 -5
  93. package/src/connection/websocket/WebsocketConnector.ts +25 -26
  94. package/src/connection/websocket/WebsocketServer.ts +27 -42
  95. package/src/connection/websocket/{ServerWebsocket.ts → WebsocketServerConnection.ts} +14 -55
  96. package/src/dht/DhtNode.ts +72 -34
  97. package/src/dht/DhtNodeRpcLocal.ts +16 -0
  98. package/src/dht/DhtNodeRpcRemote.ts +18 -0
  99. package/src/dht/ExternalApiRpcLocal.ts +5 -5
  100. package/src/dht/ExternalApiRpcRemote.ts +4 -4
  101. package/src/dht/PeerManager.ts +48 -12
  102. package/src/dht/contact/RingContactList.ts +151 -0
  103. package/src/dht/contact/ringIdentifiers.ts +62 -0
  104. package/src/dht/discovery/PeerDiscovery.ts +37 -0
  105. package/src/dht/discovery/RingDiscoverySession.ts +160 -0
  106. package/src/dht/recursive-operation/RecursiveOperationSession.ts +1 -3
  107. package/src/dht/store/StoreManager.ts +1 -1
  108. package/src/exports.ts +1 -0
  109. package/src/helpers/createPeerDescriptor.ts +2 -1
  110. package/src/helpers/version.ts +32 -0
  111. package/src/proto/packages/dht/protos/DhtRpc.client.ts +22 -9
  112. package/src/proto/packages/dht/protos/DhtRpc.server.ts +10 -4
  113. package/src/proto/packages/dht/protos/DhtRpc.ts +122 -100
  114. package/src/transport/RoutingRpcCommunicator.ts +1 -2
  115. package/test/benchmark/Find.test.ts +3 -4
  116. package/test/benchmark/KademliaCorrectness.test.ts +14 -8
  117. package/test/benchmark/RingCorrectness.test.ts +157 -0
  118. package/test/benchmark/hybrid-network-simulation/RingContactList.test.ts +72 -0
  119. package/test/data/generateGroundTruthData.ts +2 -2
  120. package/test/end-to-end/memory-leak.test.ts +1 -2
  121. package/test/integration/ConnectionManager.test.ts +28 -10
  122. package/test/integration/ConnectivityChecking.test.ts +3 -15
  123. package/test/integration/DhtNodeExternalAPI.test.ts +6 -6
  124. package/test/integration/Find.test.ts +4 -4
  125. package/test/integration/Layer1-scale.test.ts +0 -1
  126. package/test/integration/ReplicateData.test.ts +1 -1
  127. package/test/integration/RouteMessage.test.ts +1 -6
  128. package/test/integration/RouterRpcRemote.test.ts +1 -3
  129. package/test/integration/SimultaneousConnections.test.ts +9 -10
  130. package/test/integration/Store.test.ts +2 -2
  131. package/test/integration/StoreAndDelete.test.ts +3 -3
  132. package/test/integration/StoreOnDhtWithThreeNodes.test.ts +5 -5
  133. package/test/integration/StoreOnDhtWithTwoNodes.test.ts +3 -3
  134. package/test/integration/WebrtcConnectionManagement.test.ts +3 -9
  135. package/test/integration/WebsocketConnectionManagement.test.ts +1 -6
  136. package/test/integration/rpc-connections-over-webrpc.test.ts +1 -2
  137. package/test/unit/PeerManager.test.ts +5 -2
  138. package/test/unit/RecursiveOperationManager.test.ts +14 -8
  139. package/test/unit/RecursiveOperationSession.test.ts +1 -1
  140. package/test/unit/Router.test.ts +0 -2
  141. package/test/unit/RoutingSession.test.ts +1 -2
  142. package/test/unit/connectivityRequestHandler.test.ts +5 -9
  143. package/test/unit/createPeerDescriptor.test.ts +12 -6
  144. package/test/unit/version.test.ts +18 -0
  145. package/test/utils/utils.ts +43 -10
  146. package/tsconfig.jest.json +2 -1
  147. package/tsconfig.node.json +2 -1
  148. package/dist/src/connection/websocket/ServerWebsocket.js.map +0 -1
  149. package/dist/src/helpers/versionCompatibility.d.ts +0 -2
  150. package/dist/src/helpers/versionCompatibility.js +0 -18
  151. package/dist/src/helpers/versionCompatibility.js.map +0 -1
  152. package/src/helpers/versionCompatibility.ts +0 -13
  153. package/test/unit/versionCompatibility.test.ts +0 -16
@@ -17,10 +17,12 @@ import { Any } from '../proto/google/protobuf/any'
17
17
  import {
18
18
  ClosestPeersRequest,
19
19
  ClosestPeersResponse,
20
+ ClosestRingPeersRequest,
21
+ ClosestRingPeersResponse,
20
22
  ConnectivityResponse,
21
23
  DataEntry,
22
- ExternalFindDataRequest,
23
- ExternalFindDataResponse,
24
+ ExternalFetchDataRequest,
25
+ ExternalFetchDataResponse,
24
26
  ExternalStoreDataRequest,
25
27
  ExternalStoreDataResponse,
26
28
  LeaveNotice,
@@ -40,18 +42,23 @@ import { ExternalApiRpcLocal } from './ExternalApiRpcLocal'
40
42
  import { ExternalApiRpcRemote } from './ExternalApiRpcRemote'
41
43
  import { PeerManager } from './PeerManager'
42
44
  import { PeerDiscovery } from './discovery/PeerDiscovery'
43
- import { RecursiveOperationManager, RecursiveOperationResult } from './recursive-operation/RecursiveOperationManager'
45
+ import { RecursiveOperationManager } from './recursive-operation/RecursiveOperationManager'
44
46
  import { Router } from './routing/Router'
45
47
  import { LocalDataStore } from './store/LocalDataStore'
46
48
  import { StoreManager } from './store/StoreManager'
47
49
  import { StoreRpcRemote } from './store/StoreRpcRemote'
48
50
  import { createPeerDescriptor } from '../helpers/createPeerDescriptor'
51
+ import { RingIdRaw } from './contact/ringIdentifiers'
52
+ import { getLocalRegion } from '@streamr/cdn-location'
53
+ import { RingContacts } from './contact/RingContactList'
49
54
 
50
55
  export interface DhtNodeEvents {
51
56
  contactAdded: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
52
57
  contactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
53
58
  randomContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
54
59
  randomContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
60
+ ringContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
61
+ ringContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
55
62
  }
56
63
 
57
64
  export interface DhtNodeOptions {
@@ -67,6 +74,7 @@ export interface DhtNodeOptions {
67
74
  storeMaxTtl?: number
68
75
  networkConnectivityTimeout?: number
69
76
  storageRedundancyFactor?: number
77
+ region?: number
70
78
 
71
79
  transport?: ITransport
72
80
  peerDescriptor?: PeerDescriptor
@@ -123,9 +131,9 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
123
131
  private peerDiscovery?: PeerDiscovery
124
132
  private peerManager?: PeerManager
125
133
  public connectionManager?: ConnectionManager
134
+ private region?: number
126
135
  private started = false
127
136
  private abortController = new AbortController()
128
-
129
137
  constructor(conf: DhtNodeOptions) {
130
138
  super()
131
139
  this.config = merge({
@@ -150,7 +158,7 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
150
158
 
151
159
  private validateConfig(): void {
152
160
  const expectedNodeIdLength = KADEMLIA_ID_LENGTH_IN_BYTES * 2
153
- if (this.config.nodeId !== undefined ) {
161
+ if (this.config.nodeId !== undefined) {
154
162
  if (!/^[0-9a-fA-F]+$/.test(this.config.nodeId)) {
155
163
  throw new Error('Invalid nodeId, the nodeId should be a hex string')
156
164
  } else if (this.config.nodeId.length !== expectedNodeIdLength) {
@@ -177,6 +185,14 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
177
185
  this.config.peerDescriptor.websocket = undefined
178
186
  }
179
187
  }
188
+ if (this.region !== undefined) {
189
+ this.region = this.config.region
190
+ } else if (this.config.peerDescriptor?.region !== undefined) {
191
+ this.region = this.config.peerDescriptor.region
192
+ } else {
193
+ this.region = await getLocalRegion()
194
+ }
195
+
180
196
  // If transport is given, do not create a ConnectionManager
181
197
  if (this.config.transport) {
182
198
  this.transport = this.config.transport
@@ -294,10 +310,12 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
294
310
  numberOfNodesPerKBucket: this.config.numberOfNodesPerKBucket,
295
311
  maxContactListSize: this.config.maxNeighborListSize,
296
312
  localNodeId: this.getNodeId(),
313
+ localPeerDescriptor: this.localPeerDescriptor!,
297
314
  connectionManager: this.connectionManager!,
298
315
  peerDiscoveryQueryBatchSize: this.config.peerDiscoveryQueryBatchSize,
299
316
  isLayer0: (this.connectionManager !== undefined),
300
- createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => this.createDhtNodeRpcRemote(peerDescriptor)
317
+ createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => this.createDhtNodeRpcRemote(peerDescriptor),
318
+ lockId: this.config.serviceId
301
319
  })
302
320
  this.peerManager.on('contactRemoved', (peerDescriptor: PeerDescriptor, activeContacts: PeerDescriptor[]) => {
303
321
  this.emit('contactRemoved', peerDescriptor, activeContacts)
@@ -311,6 +329,12 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
311
329
  this.peerManager.on('randomContactAdded', (peerDescriptor: PeerDescriptor, activeContacts: PeerDescriptor[]) =>
312
330
  this.emit('randomContactAdded', peerDescriptor, activeContacts)
313
331
  )
332
+ this.peerManager.on('ringContactRemoved', (peerDescriptor: PeerDescriptor, activeContacts: RingContacts) => {
333
+ this.emit('ringContactRemoved', peerDescriptor, activeContacts)
334
+ })
335
+ this.peerManager.on('ringContactAdded', (peerDescriptor: PeerDescriptor, activeContacts: RingContacts) => {
336
+ this.emit('ringContactAdded', peerDescriptor, activeContacts)
337
+ })
314
338
  this.peerManager.on('kBucketEmpty', () => {
315
339
  if (!this.peerDiscovery!.isJoinOngoing()
316
340
  && this.config.entryPoints
@@ -349,26 +373,31 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
349
373
  return this.peerManager!.getClosestNeighborsTo(nodeId, limit)
350
374
  .map((dhtPeer: DhtNodeRpcRemote) => dhtPeer.getPeerDescriptor())
351
375
  },
376
+ getClosestRingPeersTo: (ringIdRaw: RingIdRaw, limit: number) => {
377
+ return this.getClosestRingContactsTo(ringIdRaw, limit)
378
+ },
352
379
  addContact: (contact: PeerDescriptor) => this.peerManager!.addContact([contact]),
353
380
  removeContact: (nodeId: DhtAddress) => this.removeContact(nodeId)
354
381
  })
355
382
  this.rpcCommunicator!.registerRpcMethod(ClosestPeersRequest, ClosestPeersResponse, 'getClosestPeers',
356
383
  (req: ClosestPeersRequest, context) => dhtNodeRpcLocal.getClosestPeers(req, context))
384
+ this.rpcCommunicator!.registerRpcMethod(ClosestRingPeersRequest, ClosestRingPeersResponse, 'getClosestRingPeers',
385
+ (req: ClosestRingPeersRequest, context) => dhtNodeRpcLocal.getClosestRingPeers(req, context))
357
386
  this.rpcCommunicator!.registerRpcMethod(PingRequest, PingResponse, 'ping',
358
387
  (req: PingRequest, context) => dhtNodeRpcLocal.ping(req, context))
359
388
  this.rpcCommunicator!.registerRpcNotification(LeaveNotice, 'leaveNotice',
360
389
  (_req: LeaveNotice, context) => dhtNodeRpcLocal.leaveNotice(context))
361
390
  const externalApiRpcLocal = new ExternalApiRpcLocal({
362
391
  executeRecursiveOperation: (key: DhtAddress, operation: RecursiveOperation, excludedPeer: DhtAddress) => {
363
- return this.executeRecursiveOperation(key, operation, excludedPeer)
392
+ return this.recursiveOperationManager!.execute(key, operation, excludedPeer)
364
393
  },
365
394
  storeDataToDht: (key: DhtAddress, data: Any, creator?: DhtAddress) => this.storeDataToDht(key, data, creator)
366
395
  })
367
396
  this.rpcCommunicator!.registerRpcMethod(
368
- ExternalFindDataRequest,
369
- ExternalFindDataResponse,
370
- 'externalFindData',
371
- (req: ExternalFindDataRequest, context: ServerCallContext) => externalApiRpcLocal.externalFindData(req, context),
397
+ ExternalFetchDataRequest,
398
+ ExternalFetchDataResponse,
399
+ 'externalFetchData',
400
+ (req: ExternalFetchDataRequest, context: ServerCallContext) => externalApiRpcLocal.externalFetchData(req, context),
372
401
  { timeout: 10000 } // TODO use config option or named constant?
373
402
  )
374
403
  this.rpcCommunicator!.registerRpcMethod(
@@ -400,7 +429,7 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
400
429
  if (this.config.peerDescriptor !== undefined) {
401
430
  this.localPeerDescriptor = this.config.peerDescriptor
402
431
  } else {
403
- this.localPeerDescriptor = createPeerDescriptor(connectivityResponse, this.config.nodeId)
432
+ this.localPeerDescriptor = createPeerDescriptor(connectivityResponse, this.region!, this.config.nodeId)
404
433
  }
405
434
  return this.localPeerDescriptor
406
435
  }
@@ -408,8 +437,15 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
408
437
  public getClosestContacts(limit?: number): PeerDescriptor[] {
409
438
  return this.peerManager!.getClosestContactsTo(
410
439
  this.getNodeId(),
411
- limit).map((peer) => peer.getPeerDescriptor()
412
- )
440
+ limit).map((peer) => peer.getPeerDescriptor())
441
+ }
442
+
443
+ public getClosestRingContactsTo(ringIdRaw: RingIdRaw, limit?: number): RingContacts {
444
+ const closest = this.peerManager!.getClosestRingContactsTo(ringIdRaw, limit)
445
+ return {
446
+ left: closest.left.map((dhtPeer: DhtNodeRpcRemote) => dhtPeer.getPeerDescriptor()),
447
+ right: closest.right.map((dhtPeer: DhtNodeRpcRemote) => dhtPeer.getPeerDescriptor())
448
+ }
413
449
  }
414
450
 
415
451
  public getNodeId(): DhtAddress {
@@ -448,25 +484,22 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
448
484
  await this.peerDiscovery!.joinDht(entryPointDescriptors, doAdditionalDistantPeerDiscovery, retry)
449
485
  }
450
486
 
451
- // TODO make this private and unify the public API of find/fetch/store/delete methods
452
- // (we already have storeDataToDht etc. here)
453
- public async executeRecursiveOperation(
454
- key: DhtAddress,
455
- operation: RecursiveOperation,
456
- excludedPeer?: DhtAddress
457
- ): Promise<RecursiveOperationResult> {
458
- return this.recursiveOperationManager!.execute(key, operation, excludedPeer)
487
+ public async joinRing(): Promise<void> {
488
+ if (!this.started) {
489
+ throw new Error('Cannot join ring before calling start() on DhtNode')
490
+ }
491
+ await this.peerDiscovery!.joinRing()
459
492
  }
460
493
 
461
494
  public async storeDataToDht(key: DhtAddress, data: Any, creator?: DhtAddress): Promise<PeerDescriptor[]> {
462
495
  const connectedEntryPoints = this.getConnectedEntryPoints()
463
496
  if (this.peerDiscovery!.isJoinOngoing() && connectedEntryPoints.length > 0) {
464
- return this.storeDataViaPeer(key, data, sample(connectedEntryPoints)!)
497
+ return this.storeDataToDhtViaPeer(key, data, sample(connectedEntryPoints)!)
465
498
  }
466
499
  return this.storeManager!.storeDataToDht(key, data, creator ?? this.getNodeId())
467
500
  }
468
501
 
469
- public async storeDataViaPeer(key: DhtAddress, data: Any, peer: PeerDescriptor): Promise<PeerDescriptor[]> {
502
+ public async storeDataToDhtViaPeer(key: DhtAddress, data: Any, peer: PeerDescriptor): Promise<PeerDescriptor[]> {
470
503
  const rpcRemote = new ExternalApiRpcRemote(
471
504
  this.localPeerDescriptor!,
472
505
  peer,
@@ -476,29 +509,34 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
476
509
  return await rpcRemote.storeData(key, data)
477
510
  }
478
511
 
479
- public async getDataFromDht(key: DhtAddress): Promise<DataEntry[]> {
512
+ public async fetchDataFromDht(key: DhtAddress): Promise<DataEntry[]> {
480
513
  const connectedEntryPoints = this.getConnectedEntryPoints()
481
514
  if (this.peerDiscovery!.isJoinOngoing() && connectedEntryPoints.length > 0) {
482
- return this.findDataViaPeer(key, sample(connectedEntryPoints)!)
515
+ return this.fetchDataFromDhtViaPeer(key, sample(connectedEntryPoints)!)
483
516
  }
484
517
  const result = await this.recursiveOperationManager!.execute(key, RecursiveOperation.FETCH_DATA)
485
518
  return result.dataEntries ?? [] // TODO is this fallback needed?
486
519
  }
487
520
 
488
- public async deleteDataFromDht(key: DhtAddress, waitForCompletion: boolean): Promise<void> {
489
- if (!this.abortController.signal.aborted) {
490
- await this.recursiveOperationManager!.execute(key, RecursiveOperation.DELETE_DATA, undefined, waitForCompletion)
491
- }
492
- }
493
-
494
- public async findDataViaPeer(key: DhtAddress, peer: PeerDescriptor): Promise<DataEntry[]> {
521
+ public async fetchDataFromDhtViaPeer(key: DhtAddress, peer: PeerDescriptor): Promise<DataEntry[]> {
495
522
  const rpcRemote = new ExternalApiRpcRemote(
496
523
  this.localPeerDescriptor!,
497
524
  peer,
498
525
  this.rpcCommunicator!,
499
526
  ExternalApiRpcClient
500
527
  )
501
- return await rpcRemote.externalFindData(key)
528
+ return await rpcRemote.externalFetchData(key)
529
+ }
530
+
531
+ public async deleteDataFromDht(key: DhtAddress, waitForCompletion: boolean): Promise<void> {
532
+ if (!this.abortController.signal.aborted) {
533
+ await this.recursiveOperationManager!.execute(key, RecursiveOperation.DELETE_DATA, undefined, waitForCompletion)
534
+ }
535
+ }
536
+
537
+ async findClosestNodesFromDht(key: DhtAddress): Promise<PeerDescriptor[]> {
538
+ const result = await this.recursiveOperationManager!.execute(key, RecursiveOperation.FIND_CLOSEST_NODES)
539
+ return result.closestNodes
502
540
  }
503
541
 
504
542
  public getTransport(): ITransport {
@@ -4,6 +4,8 @@ import { Empty } from '../proto/google/protobuf/empty'
4
4
  import {
5
5
  ClosestPeersRequest,
6
6
  ClosestPeersResponse,
7
+ ClosestRingPeersRequest,
8
+ ClosestRingPeersResponse,
7
9
  PeerDescriptor,
8
10
  PingRequest,
9
11
  PingResponse
@@ -11,10 +13,13 @@ import {
11
13
  import { IDhtNodeRpc } from '../proto/packages/dht/protos/DhtRpc.server'
12
14
  import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
13
15
  import { DhtAddress, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor } from '../identifiers'
16
+ import { RingIdRaw } from './contact/ringIdentifiers'
17
+ import { RingContacts } from './contact/RingContactList'
14
18
 
15
19
  interface DhtNodeRpcLocalConfig {
16
20
  peerDiscoveryQueryBatchSize: number
17
21
  getClosestPeersTo: (nodeId: DhtAddress, limit: number) => PeerDescriptor[]
22
+ getClosestRingPeersTo: (id: RingIdRaw, limit: number) => RingContacts
18
23
  addContact: (contact: PeerDescriptor) => void
19
24
  removeContact: (nodeId: DhtAddress) => void
20
25
  }
@@ -38,6 +43,17 @@ export class DhtNodeRpcLocal implements IDhtNodeRpc {
38
43
  return response
39
44
  }
40
45
 
46
+ async getClosestRingPeers(request: ClosestRingPeersRequest, context: ServerCallContext): Promise<ClosestRingPeersResponse> {
47
+ this.config.addContact((context as DhtCallContext).incomingSourceDescriptor!)
48
+ const closestPeers = this.config.getClosestRingPeersTo(request.ringId as RingIdRaw, this.config.peerDiscoveryQueryBatchSize)
49
+ const response = {
50
+ leftPeers: closestPeers.left,
51
+ rightPeers: closestPeers.right,
52
+ requestId: request.requestId
53
+ }
54
+ return response
55
+ }
56
+
41
57
  async ping(request: PingRequest, context: ServerCallContext): Promise<PingResponse> {
42
58
  logger.trace('received ping request: ' + getNodeIdFromPeerDescriptor((context as DhtCallContext).incomingSourceDescriptor!))
43
59
  setImmediate(() => {
@@ -4,6 +4,7 @@ import { v4 } from 'uuid'
4
4
  import { DhtAddress, DhtAddressRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../identifiers'
5
5
  import {
6
6
  ClosestPeersRequest,
7
+ ClosestRingPeersRequest,
7
8
  PeerDescriptor,
8
9
  PingRequest
9
10
  } from '../proto/packages/dht/protos/DhtRpc'
@@ -11,6 +12,8 @@ import { DhtNodeRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
11
12
  import { ServiceID } from '../types/ServiceID'
12
13
  import { RpcRemote } from './contact/RpcRemote'
13
14
  import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
15
+ import { RingIdRaw } from './contact/ringIdentifiers'
16
+ import { RingContacts } from './contact/RingContactList'
14
17
 
15
18
  const logger = new Logger(module)
16
19
 
@@ -55,6 +58,21 @@ export class DhtNodeRpcRemote extends RpcRemote<DhtNodeRpcClient> implements KBu
55
58
  }
56
59
  }
57
60
 
61
+ async getClosestRingPeers(ringIdRaw: RingIdRaw): Promise<RingContacts> {
62
+ logger.trace(`Requesting getClosestRingPeers on ${this.serviceId} from ${this.getNodeId()}`)
63
+ const request: ClosestRingPeersRequest = {
64
+ ringId: ringIdRaw,
65
+ requestId: v4()
66
+ }
67
+ try {
68
+ const response = await this.getClient().getClosestRingPeers(request, this.formDhtRpcOptions())
69
+ return { left: response.leftPeers ?? [], right: response.rightPeers ?? [] }
70
+ } catch (err) {
71
+ logger.trace(`getClosestRingPeers error ${this.serviceId}`, { err })
72
+ throw err
73
+ }
74
+ }
75
+
58
76
  async ping(): Promise<boolean> {
59
77
  logger.trace(`Requesting ping on ${this.serviceId} from ${this.getNodeId()}`)
60
78
  const request: PingRequest = {
@@ -1,7 +1,7 @@
1
1
  import { IExternalApiRpc } from '../proto/packages/dht/protos/DhtRpc.server'
2
2
  import {
3
- ExternalFindDataRequest,
4
- ExternalFindDataResponse,
3
+ ExternalFetchDataRequest,
4
+ ExternalFetchDataResponse,
5
5
  ExternalStoreDataRequest,
6
6
  ExternalStoreDataResponse,
7
7
  RecursiveOperation,
@@ -35,14 +35,14 @@ export class ExternalApiRpcLocal implements IExternalApiRpc {
35
35
  this.config = config
36
36
  }
37
37
 
38
- async externalFindData(findDataRequest: ExternalFindDataRequest, context: ServerCallContext): Promise<ExternalFindDataResponse> {
38
+ async externalFetchData(request: ExternalFetchDataRequest, context: ServerCallContext): Promise<ExternalFetchDataResponse> {
39
39
  const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
40
40
  const result = await this.config.executeRecursiveOperation(
41
- getDhtAddressFromRaw(findDataRequest.key),
41
+ getDhtAddressFromRaw(request.key),
42
42
  RecursiveOperation.FETCH_DATA,
43
43
  getNodeIdFromPeerDescriptor(senderPeerDescriptor)
44
44
  )
45
- return ExternalFindDataResponse.create({ entries: result.dataEntries ?? [] })
45
+ return ExternalFetchDataResponse.create({ entries: result.dataEntries ?? [] })
46
46
  }
47
47
 
48
48
  async externalStoreData(request: ExternalStoreDataRequest, context: ServerCallContext): Promise<ExternalStoreDataResponse> {
@@ -1,13 +1,13 @@
1
1
  import { DhtAddress, getRawFromDhtAddress } from '../identifiers'
2
2
  import { Any } from '../proto/google/protobuf/any'
3
- import { DataEntry, ExternalFindDataRequest, ExternalStoreDataRequest, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
3
+ import { DataEntry, ExternalFetchDataRequest, ExternalStoreDataRequest, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
4
4
  import { ExternalApiRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
5
5
  import { RpcRemote } from './contact/RpcRemote'
6
6
 
7
7
  export class ExternalApiRpcRemote extends RpcRemote<ExternalApiRpcClient> {
8
8
 
9
- async externalFindData(key: DhtAddress): Promise<DataEntry[]> {
10
- const request: ExternalFindDataRequest = {
9
+ async externalFetchData(key: DhtAddress): Promise<DataEntry[]> {
10
+ const request: ExternalFetchDataRequest = {
11
11
  key: getRawFromDhtAddress(key)
12
12
  }
13
13
  const options = this.formDhtRpcOptions({
@@ -15,7 +15,7 @@ export class ExternalApiRpcRemote extends RpcRemote<ExternalApiRpcClient> {
15
15
  timeout: 10000
16
16
  })
17
17
  try {
18
- const data = await this.getClient().externalFindData(request, options)
18
+ const data = await this.getClient().externalFetchData(request, options)
19
19
  return data.entries
20
20
  } catch (err) {
21
21
  return []
@@ -11,6 +11,9 @@ import { SortedContactList } from './contact/SortedContactList'
11
11
  import { ConnectionManager } from '../connection/ConnectionManager'
12
12
  import EventEmitter from 'eventemitter3'
13
13
  import { DhtAddress, DhtAddressRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../identifiers'
14
+ import { RingContactList, RingContacts } from './contact/RingContactList'
15
+ import { RingIdRaw, getRingIdRawFromPeerDescriptor } from './contact/ringIdentifiers'
16
+ import { LockID } from '../connection/ConnectionLockHandler'
14
17
 
15
18
  const logger = new Logger(module)
16
19
 
@@ -19,8 +22,10 @@ interface PeerManagerConfig {
19
22
  maxContactListSize: number
20
23
  peerDiscoveryQueryBatchSize: number
21
24
  localNodeId: DhtAddress
25
+ localPeerDescriptor: PeerDescriptor
22
26
  connectionManager: ConnectionManager
23
27
  isLayer0: boolean
28
+ lockId: LockID
24
29
  createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => DhtNodeRpcRemote
25
30
  }
26
31
 
@@ -29,6 +34,8 @@ export interface PeerManagerEvents {
29
34
  contactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
30
35
  randomContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
31
36
  randomContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
37
+ ringContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
38
+ ringContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
32
39
  kBucketEmpty: () => void
33
40
  }
34
41
 
@@ -51,6 +58,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
51
58
  // All nodes that we know about
52
59
  private contacts: SortedContactList<DhtNodeRpcRemote>
53
60
  private randomPeers: RandomContactList<DhtNodeRpcRemote>
61
+ private ringContacts: RingContactList<DhtNodeRpcRemote>
54
62
  private stopped: boolean = false
55
63
  private readonly config: PeerManagerConfig
56
64
 
@@ -62,6 +70,13 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
62
70
  numberOfNodesPerKBucket: this.config.numberOfNodesPerKBucket,
63
71
  numberOfNodesToPing: this.config.numberOfNodesPerKBucket
64
72
  })
73
+ this.ringContacts = new RingContactList<DhtNodeRpcRemote>(getRingIdRawFromPeerDescriptor(this.config.localPeerDescriptor), true)
74
+ this.ringContacts.on('ringContactAdded', (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => {
75
+ this.emit('ringContactAdded', peerDescriptor, closestPeers)
76
+ })
77
+ this.ringContacts.on('ringContactRemoved', (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => {
78
+ this.emit('ringContactRemoved', peerDescriptor, closestPeers)
79
+ })
65
80
  this.bucket.on('ping', (oldContacts: DhtNodeRpcRemote[], newContact: DhtNodeRpcRemote) => this.onKBucketPing(oldContacts, newContact))
66
81
  this.bucket.on('removed', (contact: DhtNodeRpcRemote) => this.onKBucketRemoved(getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())))
67
82
  this.bucket.on('added', (contact: DhtNodeRpcRemote) => this.onKBucketAdded(contact))
@@ -69,7 +84,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
69
84
  // TODO: Update contact info to the connection manager and reconnect
70
85
  })
71
86
  this.contacts = new SortedContactList({
72
- referenceId: this.config.localNodeId,
87
+ referenceId: this.config.localNodeId,
73
88
  maxSize: this.config.maxContactListSize,
74
89
  allowToContainReferenceId: false,
75
90
  emitEvents: true
@@ -98,7 +113,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
98
113
  return
99
114
  }
100
115
  const sortingList: SortedContactList<DhtNodeRpcRemote> = new SortedContactList({
101
- referenceId: this.config.localNodeId,
116
+ referenceId: this.config.localNodeId,
102
117
  maxSize: 100, // TODO use config option or named constant?
103
118
  allowToContainReferenceId: false,
104
119
  emitEvents: false
@@ -106,7 +121,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
106
121
  sortingList.addContacts(oldContacts)
107
122
  const sortedContacts = sortingList.getAllContacts()
108
123
  const removableNodeId = sortedContacts[sortedContacts.length - 1].getNodeId()
109
- this.config.connectionManager?.weakUnlockConnection(removableNodeId)
124
+ this.config.connectionManager?.weakUnlockConnection(removableNodeId, this.config.lockId)
110
125
  this.bucket.remove(getRawFromDhtAddress(removableNodeId))
111
126
  this.bucket.add(newContact)
112
127
  }
@@ -115,7 +130,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
115
130
  if (this.stopped) {
116
131
  return
117
132
  }
118
- this.config.connectionManager?.weakUnlockConnection(nodeId)
133
+ this.config.connectionManager?.weakUnlockConnection(nodeId, this.config.lockId)
119
134
  logger.trace(`Removed contact ${nodeId}`)
120
135
  if (this.bucket.count() === 0) {
121
136
  this.emit('kBucketEmpty')
@@ -130,7 +145,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
130
145
  const peerDescriptor = contact.getPeerDescriptor()
131
146
  const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
132
147
  // Important to lock here, before the ping result is known
133
- this.config.connectionManager?.weakLockConnection(nodeId)
148
+ this.config.connectionManager?.weakLockConnection(nodeId, this.config.lockId)
134
149
  if (this.connections.has(contact.getNodeId())) {
135
150
  logger.trace(`Added new contact ${nodeId}`)
136
151
  } else { // open connection by pinging
@@ -140,13 +155,13 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
140
155
  logger.trace(`Added new contact ${nodeId}`)
141
156
  } else {
142
157
  logger.trace('ping failed ' + nodeId)
143
- this.config.connectionManager?.weakUnlockConnection(nodeId)
158
+ this.config.connectionManager?.weakUnlockConnection(nodeId, this.config.lockId)
144
159
  this.removeContact(nodeId)
145
160
  this.addClosestContactToBucket()
146
161
  }
147
162
  return
148
163
  }).catch((_e) => {
149
- this.config.connectionManager?.weakUnlockConnection(nodeId)
164
+ this.config.connectionManager?.weakUnlockConnection(nodeId, this.config.lockId)
150
165
  this.removeContact(nodeId)
151
166
  this.addClosestContactToBucket()
152
167
  })
@@ -207,6 +222,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
207
222
  return
208
223
  }
209
224
  logger.trace(`Removing contact ${nodeId}`)
225
+ this.ringContacts.removeContact(this.contacts.getContact(nodeId)?.contact)
210
226
  this.bucket.remove(getRawFromDhtAddress(nodeId))
211
227
  this.contacts.removeContact(nodeId)
212
228
  this.randomPeers.removeContact(nodeId)
@@ -219,6 +235,10 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
219
235
  this.bucket.remove(rpcRemote.id)
220
236
  })
221
237
  this.bucket.removeAllListeners()
238
+ this.ringContacts.getAllContacts().forEach((rpcRemote) => {
239
+ rpcRemote.leaveNotice()
240
+ this.ringContacts.removeContact(rpcRemote)
241
+ })
222
242
  this.contacts.stop()
223
243
  this.randomPeers.stop()
224
244
  this.connections.clear()
@@ -230,7 +250,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
230
250
  allowToContainReferenceId: true,
231
251
  emitEvents: false,
232
252
  excludedNodeIds
233
- })
253
+ })
234
254
  this.bucket.toArray().forEach((contact) => closest.addContact(contact))
235
255
  return closest.getClosestContacts(limit)
236
256
  }
@@ -248,13 +268,24 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
248
268
  return closest.getClosestContacts(limit)
249
269
  }
250
270
 
271
+ getClosestRingContactsTo(
272
+ ringIdRaw: RingIdRaw,
273
+ limit?: number,
274
+ excludedIds?: Set<DhtAddress>
275
+ ): { left: DhtNodeRpcRemote[], right: DhtNodeRpcRemote[] } {
276
+ const closest = new RingContactList<DhtNodeRpcRemote>(ringIdRaw, false, excludedIds)
277
+ this.contacts.getAllContacts().map((contact) => closest.addContact(contact))
278
+ this.ringContacts.getAllContacts().map((contact) => closest.addContact(contact))
279
+ return closest.getClosestContacts(limit ?? 8)
280
+ }
281
+
251
282
  getContactCount(excludedNodeIds?: Set<DhtAddress>): number {
252
283
  return this.contacts.getAllContacts().filter((contact) => {
253
284
  if (!excludedNodeIds) {
254
285
  return true
255
286
  } else {
256
287
  return !excludedNodeIds.has(contact.getNodeId())
257
- }
288
+ }
258
289
  }).length
259
290
  }
260
291
 
@@ -274,7 +305,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
274
305
  this.contacts.setActive(nodeId)
275
306
  }
276
307
 
277
- addContact(peerDescriptors: PeerDescriptor[], setActive?: boolean): void {
308
+ addContact(peerDescriptors: PeerDescriptor[], setActive?: boolean): void {
278
309
  if (this.stopped) {
279
310
  return
280
311
  }
@@ -285,18 +316,23 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
285
316
  const remote = this.config.createDhtNodeRpcRemote(contact)
286
317
  const isInBucket = (this.bucket.get(contact.nodeId) !== null)
287
318
  const isInContacts = (this.contacts.getContact(nodeId) !== undefined)
319
+ const isInRingContacts = this.ringContacts.getContact(contact) !== undefined
320
+
288
321
  if (isInBucket || isInContacts) {
289
322
  this.randomPeers.addContact(remote)
290
323
  }
291
324
  if (!isInBucket) {
292
325
  this.bucket.add(remote)
293
- }
326
+ }
294
327
  if (!isInContacts) {
295
328
  this.contacts.addContact(remote)
296
- }
329
+ }
297
330
  if (setActive) {
298
331
  this.contacts.setActive(nodeId)
299
332
  }
333
+ if (!isInRingContacts) {
334
+ this.ringContacts.addContact(remote)
335
+ }
300
336
  }
301
337
  })
302
338
  }