@streamr/dht 100.0.0-testnet-three.6 → 100.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +1 -1
  2. package/dist/package.json +12 -8
  3. package/dist/src/connection/ConnectionLockHandler.d.ts +2 -2
  4. package/dist/src/connection/ConnectionLockHandler.js +13 -5
  5. package/dist/src/connection/ConnectionLockHandler.js.map +1 -1
  6. package/dist/src/connection/ConnectionManager.d.ts +4 -4
  7. package/dist/src/connection/ConnectionManager.js +8 -7
  8. package/dist/src/connection/ConnectionManager.js.map +1 -1
  9. package/dist/src/connection/ConnectorFacade.js.map +1 -1
  10. package/dist/src/connection/Handshaker.js +6 -13
  11. package/dist/src/connection/Handshaker.js.map +1 -1
  12. package/dist/src/connection/connectivityChecker.d.ts +1 -1
  13. package/dist/src/connection/connectivityChecker.js +5 -5
  14. package/dist/src/connection/connectivityChecker.js.map +1 -1
  15. package/dist/src/connection/connectivityRequestHandler.d.ts +2 -2
  16. package/dist/src/connection/connectivityRequestHandler.js +6 -7
  17. package/dist/src/connection/connectivityRequestHandler.js.map +1 -1
  18. package/dist/src/connection/webrtc/WebrtcConnector.d.ts +1 -0
  19. package/dist/src/connection/webrtc/WebrtcConnector.js +13 -8
  20. package/dist/src/connection/webrtc/WebrtcConnector.js.map +1 -1
  21. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.d.ts +2 -0
  22. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js +4 -6
  23. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js.map +1 -1
  24. package/dist/src/connection/websocket/WebsocketConnector.js +24 -21
  25. package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
  26. package/dist/src/connection/websocket/WebsocketServer.js +25 -35
  27. package/dist/src/connection/websocket/WebsocketServer.js.map +1 -1
  28. package/dist/src/connection/websocket/{ServerWebsocket.d.ts → WebsocketServerConnection.d.ts} +4 -5
  29. package/dist/src/connection/websocket/{ServerWebsocket.js → WebsocketServerConnection.js} +15 -48
  30. package/dist/src/connection/websocket/WebsocketServerConnection.js.map +1 -0
  31. package/dist/src/dht/DhtNode.d.ts +13 -6
  32. package/dist/src/dht/DhtNode.js +50 -15
  33. package/dist/src/dht/DhtNode.js.map +1 -1
  34. package/dist/src/dht/DhtNodeRpcLocal.d.ts +5 -1
  35. package/dist/src/dht/DhtNodeRpcLocal.js +10 -0
  36. package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
  37. package/dist/src/dht/DhtNodeRpcRemote.d.ts +3 -0
  38. package/dist/src/dht/DhtNodeRpcRemote.js +15 -0
  39. package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
  40. package/dist/src/dht/ExternalApiRpcLocal.d.ts +2 -2
  41. package/dist/src/dht/ExternalApiRpcLocal.js +3 -3
  42. package/dist/src/dht/ExternalApiRpcLocal.js.map +1 -1
  43. package/dist/src/dht/ExternalApiRpcRemote.d.ts +1 -1
  44. package/dist/src/dht/ExternalApiRpcRemote.js +2 -2
  45. package/dist/src/dht/ExternalApiRpcRemote.js.map +1 -1
  46. package/dist/src/dht/PeerManager.d.ts +12 -0
  47. package/dist/src/dht/PeerManager.js +30 -5
  48. package/dist/src/dht/PeerManager.js.map +1 -1
  49. package/dist/src/dht/contact/RingContactList.d.ts +31 -0
  50. package/dist/src/dht/contact/RingContactList.js +133 -0
  51. package/dist/src/dht/contact/RingContactList.js.map +1 -0
  52. package/dist/src/dht/contact/ringIdentifiers.d.ts +16 -0
  53. package/dist/src/dht/contact/ringIdentifiers.js +54 -0
  54. package/dist/src/dht/contact/ringIdentifiers.js.map +1 -0
  55. package/dist/src/dht/discovery/PeerDiscovery.d.ts +4 -0
  56. package/dist/src/dht/discovery/PeerDiscovery.js +35 -0
  57. package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
  58. package/dist/src/dht/discovery/RingDiscoverySession.d.ts +29 -0
  59. package/dist/src/dht/discovery/RingDiscoverySession.js +123 -0
  60. package/dist/src/dht/discovery/RingDiscoverySession.js.map +1 -0
  61. package/dist/src/dht/recursive-operation/RecursiveOperationSession.js +0 -1
  62. package/dist/src/dht/recursive-operation/RecursiveOperationSession.js.map +1 -1
  63. package/dist/src/dht/store/StoreManager.js +1 -1
  64. package/dist/src/dht/store/StoreManager.js.map +1 -1
  65. package/dist/src/exports.d.ts +1 -0
  66. package/dist/src/exports.js.map +1 -1
  67. package/dist/src/helpers/createPeerDescriptor.d.ts +1 -1
  68. package/dist/src/helpers/createPeerDescriptor.js +2 -1
  69. package/dist/src/helpers/createPeerDescriptor.js.map +1 -1
  70. package/dist/src/helpers/version.d.ts +6 -0
  71. package/dist/src/helpers/version.js +38 -0
  72. package/dist/src/helpers/version.js.map +1 -0
  73. package/dist/src/proto/packages/dht/protos/DhtRpc.client.d.ts +16 -6
  74. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js +11 -4
  75. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js.map +1 -1
  76. package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +98 -87
  77. package/dist/src/proto/packages/dht/protos/DhtRpc.js +45 -49
  78. package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
  79. package/dist/src/proto/packages/dht/protos/DhtRpc.server.d.ts +10 -4
  80. package/dist/src/transport/RoutingRpcCommunicator.js +0 -2
  81. package/dist/src/transport/RoutingRpcCommunicator.js.map +1 -1
  82. package/package.json +12 -8
  83. package/protos/DhtRpc.proto +21 -21
  84. package/src/connection/ConnectionLockHandler.ts +13 -5
  85. package/src/connection/ConnectionManager.ts +10 -10
  86. package/src/connection/ConnectorFacade.ts +0 -1
  87. package/src/connection/Handshaker.ts +7 -15
  88. package/src/connection/connectivityChecker.ts +6 -7
  89. package/src/connection/connectivityRequestHandler.ts +12 -12
  90. package/src/connection/webrtc/NodeWebrtcConnection.ts +1 -1
  91. package/src/connection/webrtc/WebrtcConnector.ts +14 -8
  92. package/src/connection/webrtc/WebrtcConnectorRpcLocal.ts +5 -5
  93. package/src/connection/websocket/WebsocketConnector.ts +25 -26
  94. package/src/connection/websocket/WebsocketServer.ts +27 -42
  95. package/src/connection/websocket/{ServerWebsocket.ts → WebsocketServerConnection.ts} +14 -55
  96. package/src/dht/DhtNode.ts +72 -34
  97. package/src/dht/DhtNodeRpcLocal.ts +16 -0
  98. package/src/dht/DhtNodeRpcRemote.ts +18 -0
  99. package/src/dht/ExternalApiRpcLocal.ts +5 -5
  100. package/src/dht/ExternalApiRpcRemote.ts +4 -4
  101. package/src/dht/PeerManager.ts +48 -12
  102. package/src/dht/contact/RingContactList.ts +151 -0
  103. package/src/dht/contact/ringIdentifiers.ts +62 -0
  104. package/src/dht/discovery/PeerDiscovery.ts +37 -0
  105. package/src/dht/discovery/RingDiscoverySession.ts +160 -0
  106. package/src/dht/recursive-operation/RecursiveOperationSession.ts +1 -3
  107. package/src/dht/store/StoreManager.ts +1 -1
  108. package/src/exports.ts +1 -0
  109. package/src/helpers/createPeerDescriptor.ts +2 -1
  110. package/src/helpers/version.ts +32 -0
  111. package/src/proto/packages/dht/protos/DhtRpc.client.ts +22 -9
  112. package/src/proto/packages/dht/protos/DhtRpc.server.ts +10 -4
  113. package/src/proto/packages/dht/protos/DhtRpc.ts +122 -100
  114. package/src/transport/RoutingRpcCommunicator.ts +1 -2
  115. package/test/benchmark/Find.test.ts +3 -4
  116. package/test/benchmark/KademliaCorrectness.test.ts +14 -8
  117. package/test/benchmark/RingCorrectness.test.ts +157 -0
  118. package/test/benchmark/hybrid-network-simulation/RingContactList.test.ts +72 -0
  119. package/test/data/generateGroundTruthData.ts +2 -2
  120. package/test/end-to-end/memory-leak.test.ts +1 -2
  121. package/test/integration/ConnectionManager.test.ts +28 -10
  122. package/test/integration/ConnectivityChecking.test.ts +3 -15
  123. package/test/integration/DhtNodeExternalAPI.test.ts +6 -6
  124. package/test/integration/Find.test.ts +4 -4
  125. package/test/integration/Layer1-scale.test.ts +0 -1
  126. package/test/integration/ReplicateData.test.ts +1 -1
  127. package/test/integration/RouteMessage.test.ts +1 -6
  128. package/test/integration/RouterRpcRemote.test.ts +1 -3
  129. package/test/integration/SimultaneousConnections.test.ts +9 -10
  130. package/test/integration/Store.test.ts +2 -2
  131. package/test/integration/StoreAndDelete.test.ts +3 -3
  132. package/test/integration/StoreOnDhtWithThreeNodes.test.ts +5 -5
  133. package/test/integration/StoreOnDhtWithTwoNodes.test.ts +3 -3
  134. package/test/integration/WebrtcConnectionManagement.test.ts +3 -9
  135. package/test/integration/WebsocketConnectionManagement.test.ts +1 -6
  136. package/test/integration/rpc-connections-over-webrpc.test.ts +1 -2
  137. package/test/unit/PeerManager.test.ts +5 -2
  138. package/test/unit/RecursiveOperationManager.test.ts +14 -8
  139. package/test/unit/RecursiveOperationSession.test.ts +1 -1
  140. package/test/unit/Router.test.ts +0 -2
  141. package/test/unit/RoutingSession.test.ts +1 -2
  142. package/test/unit/connectivityRequestHandler.test.ts +5 -9
  143. package/test/unit/createPeerDescriptor.test.ts +12 -6
  144. package/test/unit/version.test.ts +18 -0
  145. package/test/utils/utils.ts +43 -10
  146. package/tsconfig.jest.json +2 -1
  147. package/tsconfig.node.json +2 -1
  148. package/dist/src/connection/websocket/ServerWebsocket.js.map +0 -1
  149. package/dist/src/helpers/versionCompatibility.d.ts +0 -2
  150. package/dist/src/helpers/versionCompatibility.js +0 -18
  151. package/dist/src/helpers/versionCompatibility.js.map +0 -1
  152. package/src/helpers/versionCompatibility.ts +0 -13
  153. package/test/unit/versionCompatibility.test.ts +0 -16
@@ -0,0 +1,151 @@
1
+ import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
2
+ import { OrderedMap } from '@js-sdsl/ordered-map'
3
+ import { RingDistance, RingId, RingIdRaw, getLeftDistance, getRightDistance, getRingIdFromPeerDescriptor, getRingIdFromRaw } from './ringIdentifiers'
4
+ import { DhtAddress, getNodeIdFromPeerDescriptor } from '../../identifiers'
5
+ import EventEmitter from 'eventemitter3'
6
+
7
+ export interface RingContacts {
8
+ left: PeerDescriptor[]
9
+ right: PeerDescriptor[]
10
+ }
11
+ export interface RingContactListEvents {
12
+ ringContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
13
+ ringContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
14
+ }
15
+ export class RingContactList<C extends { getPeerDescriptor(): PeerDescriptor }> extends EventEmitter<RingContactListEvents> {
16
+
17
+ private readonly numNeighborsPerSide = 5
18
+ private readonly referenceId: RingId
19
+ private readonly excludedIds: Set<DhtAddress>
20
+ private readonly leftNeighbors: OrderedMap<RingDistance, C>
21
+ private readonly rightNeighbors: OrderedMap<RingDistance, C>
22
+ private readonly emitEvents: boolean
23
+
24
+ constructor(rawReferenceId: RingIdRaw, emitEvents: boolean, excludedIds?: Set<DhtAddress>) {
25
+ super()
26
+ this.referenceId = getRingIdFromRaw(rawReferenceId)
27
+ this.emitEvents = emitEvents
28
+ this.excludedIds = excludedIds ?? new Set()
29
+ this.leftNeighbors = new OrderedMap<RingDistance, C>()
30
+ this.rightNeighbors = new OrderedMap<RingDistance, C>()
31
+ }
32
+
33
+ addContact(contact: C): void {
34
+ const id = getRingIdFromPeerDescriptor(contact.getPeerDescriptor())
35
+ if (id === this.referenceId || this.excludedIds.has(getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))) {
36
+ return
37
+ }
38
+ let elementAdded = false
39
+ let elementRemoved = false
40
+
41
+ const leftDistance = getLeftDistance(this.referenceId, id)
42
+ const lastLeftNeighbor = this.leftNeighbors.back()
43
+ if (lastLeftNeighbor === undefined || leftDistance < lastLeftNeighbor[0]) {
44
+ this.leftNeighbors.setElement(leftDistance, contact)
45
+ elementAdded = true
46
+ if (this.leftNeighbors.size() > this.numNeighborsPerSide) {
47
+ this.leftNeighbors.eraseElementByIterator(this.leftNeighbors.rBegin())
48
+ elementRemoved = true
49
+ }
50
+ }
51
+
52
+ const rightDistance = getRightDistance(this.referenceId, id)
53
+ const lastRightNeighbor = this.rightNeighbors.back()
54
+ if (lastRightNeighbor === undefined || rightDistance < lastRightNeighbor[0]) {
55
+ this.rightNeighbors.setElement(rightDistance, contact)
56
+ elementAdded = true
57
+ if (this.rightNeighbors.size() > this.numNeighborsPerSide) {
58
+ this.rightNeighbors.eraseElementByIterator(this.rightNeighbors.rBegin())
59
+ elementRemoved = true
60
+ }
61
+ }
62
+
63
+ if (this.emitEvents && (elementAdded || elementRemoved)) {
64
+ const closestContacts = this.getClosestContacts()
65
+ const closestDescriptors = {
66
+ left: closestContacts.left.map((c) => c.getPeerDescriptor()),
67
+ right: closestContacts.right.map((c) => c.getPeerDescriptor())
68
+ }
69
+ if (elementAdded) {
70
+ this.emit('ringContactAdded', contact.getPeerDescriptor(), closestDescriptors)
71
+ }
72
+ if (elementRemoved) {
73
+ this.emit('ringContactRemoved', contact.getPeerDescriptor(), closestDescriptors)
74
+ }
75
+ }
76
+ }
77
+
78
+ removeContact(contact?: C): void {
79
+ if (contact === undefined) {
80
+ return
81
+ }
82
+
83
+ const id = getRingIdFromPeerDescriptor(contact.getPeerDescriptor())
84
+ const leftDistance = getLeftDistance(this.referenceId, id)
85
+ const rightDistance = getRightDistance(this.referenceId, id)
86
+
87
+ let elementRemoved = false
88
+ if (this.leftNeighbors.eraseElementByKey(leftDistance)) {
89
+ elementRemoved = true
90
+ }
91
+ if (this.rightNeighbors.eraseElementByKey(rightDistance)) {
92
+ elementRemoved = true
93
+ }
94
+
95
+ if (this.emitEvents && elementRemoved) {
96
+ const closestContacts = this.getClosestContacts()
97
+ const closestDescriptors = { left: closestContacts.left.map((c) => c.getPeerDescriptor()),
98
+ right: closestContacts.right.map((c) => c.getPeerDescriptor()) }
99
+ this.emit('ringContactRemoved', contact.getPeerDescriptor(), closestDescriptors)
100
+ }
101
+ }
102
+
103
+ getContact(peerDescriptor: PeerDescriptor): C | undefined {
104
+ const id = getRingIdFromPeerDescriptor(peerDescriptor)
105
+ const leftDistance = getLeftDistance(this.referenceId, id)
106
+ const rightDistance = getRightDistance(this.referenceId, id)
107
+ if (this.leftNeighbors.getElementByKey(leftDistance)) {
108
+ return this.leftNeighbors.getElementByKey(leftDistance)
109
+ }
110
+ if (this.rightNeighbors.getElementByKey(rightDistance)) {
111
+ return this.rightNeighbors.getElementByKey(rightDistance)
112
+ }
113
+ return undefined
114
+ }
115
+
116
+ getClosestContacts(limitPerSide?: number): { left: C[], right: C[] } {
117
+ const leftContacts: C[] = []
118
+ const rightContacts: C[] = []
119
+
120
+ let leftCount = 0
121
+ for (const item of this.leftNeighbors) {
122
+ if (limitPerSide != undefined && leftCount >= limitPerSide) {
123
+ break
124
+ }
125
+ leftContacts.push(item[1])
126
+ leftCount++
127
+ }
128
+
129
+ let rightCount = 0
130
+ for (const item of this.rightNeighbors) {
131
+ if (limitPerSide != undefined && rightCount >= limitPerSide) {
132
+ break
133
+ }
134
+ rightContacts.push(item[1])
135
+ rightCount++
136
+ }
137
+
138
+ return { left: leftContacts, right: rightContacts }
139
+ }
140
+
141
+ getAllContacts(): C[] {
142
+ const ret: C[] = []
143
+ for (const item of this.leftNeighbors) {
144
+ ret.push(item[1])
145
+ }
146
+ for (const item of this.rightNeighbors) {
147
+ ret.push(item[1])
148
+ }
149
+ return ret
150
+ }
151
+ }
@@ -0,0 +1,62 @@
1
+ import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
2
+
3
+ // Notice: you cannot convert RingId to RingIdRaw, because
4
+ // RingId is only an approximation of the actual value.
5
+ // That is why RingIdRaw is widely used in the codebase.
6
+
7
+ export type RingIdRaw = Uint8Array & { __ringIdRaw: never }
8
+ export type RingId = number & { __ringId: never }
9
+ export type RingDistance = number & { __ringDistance: never }
10
+
11
+ export const RING_SIZE = 2 ** 120 - 1 // 2^120 - 1
12
+
13
+ const binaryToBigInt = (binary: Uint8Array): bigint => {
14
+ return binary.reduce((acc, val) => (acc << BigInt(8)) | BigInt(val), BigInt(0))
15
+ }
16
+
17
+ export const getRingIdFromRaw = (raw: RingIdRaw): RingId => Number(binaryToBigInt(raw)) as RingId
18
+
19
+ export const getRingIdRawFromPeerDescriptor = (peerDescriptor: PeerDescriptor): RingIdRaw => {
20
+ const regionAsBuffer = Buffer.alloc(4)
21
+ regionAsBuffer.writeUInt32BE(peerDescriptor.region ?? 0, 0)
22
+ const ipAsbuffer = Buffer.alloc(4)
23
+ ipAsbuffer.writeUInt32BE(peerDescriptor.ipAddress ?? 0, 0)
24
+
25
+ const uniquePartAsBuffer = Buffer.from(peerDescriptor.nodeId.subarray(peerDescriptor.nodeId.length - 7, peerDescriptor.nodeId.length))
26
+
27
+ const arr = [
28
+ regionAsBuffer,
29
+ ipAsbuffer,
30
+ uniquePartAsBuffer
31
+ ]
32
+
33
+ const buffer = Buffer.concat(arr)
34
+ return new Uint8Array(buffer) as RingIdRaw
35
+ }
36
+
37
+ export const getRingIdFromPeerDescriptor = (peerDescriptor: PeerDescriptor): RingId => {
38
+ const raw = getRingIdRawFromPeerDescriptor(peerDescriptor)
39
+ return Number(binaryToBigInt(raw)) as RingId
40
+ }
41
+
42
+ export const getLeftDistance = (referenceId: RingId, id: RingId): RingDistance =>{
43
+ const diff = Math.abs(referenceId - id)
44
+ if (referenceId > id) {
45
+ // if id is smaller than referenceId, then the distance is the difference
46
+ return diff as RingDistance
47
+ } else {
48
+ // if id is bigger than referenceId, then the distance is the ringSize - difference
49
+ return RING_SIZE - diff as RingDistance
50
+ }
51
+ }
52
+
53
+ export const getRightDistance = (referenceId: RingId, id: RingId): RingDistance => {
54
+ const diff = Math.abs(referenceId - id)
55
+ if (referenceId > id) {
56
+ // if id is smaller than referenceId, then the distance is the ringSize - difference
57
+ return RING_SIZE - diff as RingDistance
58
+ } else {
59
+ // if id is bigger than referenceId, then the distance is the difference
60
+ return diff as RingDistance
61
+ }
62
+ }
@@ -6,6 +6,8 @@ import { ConnectionManager } from '../../connection/ConnectionManager'
6
6
  import { PeerManager } from '../PeerManager'
7
7
  import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
8
8
  import { ServiceID } from '../../types/ServiceID'
9
+ import { RingDiscoverySession } from './RingDiscoverySession'
10
+ import { RingIdRaw, getRingIdRawFromPeerDescriptor } from '../contact/ringIdentifiers'
9
11
 
10
12
  interface PeerDiscoveryConfig {
11
13
  localPeerDescriptor: PeerDescriptor
@@ -29,6 +31,8 @@ const logger = new Logger(module)
29
31
  export class PeerDiscovery {
30
32
 
31
33
  private ongoingDiscoverySessions: Map<string, DiscoverySession> = new Map()
34
+ private ongoingRingDiscoverySessions: Map<string, RingDiscoverySession> = new Map()
35
+
32
36
  private rejoinOngoing = false
33
37
  private joinCalled = false
34
38
  private readonly abortController: AbortController
@@ -86,6 +90,12 @@ export class PeerDiscovery {
86
90
 
87
91
  }
88
92
 
93
+ async joinRing(): Promise<void> {
94
+ const contactedPeers = new Set<DhtAddress>()
95
+ const sessions = [this.createRingSession(getRingIdRawFromPeerDescriptor(this.config.localPeerDescriptor), contactedPeers)]
96
+ await this.runRingSessions(sessions)
97
+ }
98
+
89
99
  private createSession(targetId: DhtAddress, contactedPeers: Set<DhtAddress>): DiscoverySession {
90
100
  const sessionOptions = {
91
101
  targetId,
@@ -97,6 +107,17 @@ export class PeerDiscovery {
97
107
  return new DiscoverySession(sessionOptions)
98
108
  }
99
109
 
110
+ private createRingSession(targetId: RingIdRaw, contactedPeers: Set<DhtAddress>): RingDiscoverySession {
111
+ const sessionOptions = {
112
+ targetId,
113
+ parallelism: this.config.parallelism,
114
+ noProgressLimit: this.config.joinNoProgressLimit,
115
+ peerManager: this.config.peerManager,
116
+ contactedPeers
117
+ }
118
+ return new RingDiscoverySession(sessionOptions)
119
+ }
120
+
100
121
  private async runSessions(sessions: DiscoverySession[], entryPointDescriptor: PeerDescriptor, retry: boolean): Promise<void> {
101
122
  try {
102
123
  for (const session of sessions) {
@@ -121,6 +142,19 @@ export class PeerDiscovery {
121
142
  }
122
143
  }
123
144
 
145
+ private async runRingSessions(sessions: RingDiscoverySession[]): Promise<void> {
146
+ try {
147
+ for (const session of sessions) {
148
+ this.ongoingRingDiscoverySessions.set(session.id, session)
149
+ await session.findClosestNodes(this.config.joinTimeout)
150
+ }
151
+ } catch (_e) {
152
+ logger.debug(`Ring join on ${this.config.serviceId} timed out`)
153
+ } finally {
154
+ sessions.forEach((session) => this.ongoingDiscoverySessions.delete(session.id))
155
+ }
156
+ }
157
+
124
158
  public async rejoinDht(entryPoint: PeerDescriptor): Promise<void> {
125
159
  if (this.isStopped() || this.rejoinOngoing) {
126
160
  return
@@ -184,5 +218,8 @@ export class PeerDiscovery {
184
218
  this.ongoingDiscoverySessions.forEach((session) => {
185
219
  session.stop()
186
220
  })
221
+ this.ongoingRingDiscoverySessions.forEach((session) => {
222
+ session.stop()
223
+ })
187
224
  }
188
225
  }
@@ -0,0 +1,160 @@
1
+ import { Logger, runAndWaitForEvents3 } from '@streamr/utils'
2
+ import EventEmitter from 'eventemitter3'
3
+ import { v4 } from 'uuid'
4
+ import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
5
+ import { PeerManager } from '../PeerManager'
6
+ import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
7
+ import { DhtAddress, getNodeIdFromPeerDescriptor } from '../../identifiers'
8
+ import { RingId, RingIdRaw, getLeftDistance, getRingIdFromPeerDescriptor, getRingIdFromRaw } from '../contact/ringIdentifiers'
9
+ import { RingContacts } from '../contact/RingContactList'
10
+
11
+ const logger = new Logger(module)
12
+
13
+ interface RingDiscoverySessionEvents {
14
+ discoveryCompleted: () => void
15
+ }
16
+
17
+ interface RingDiscoverySessionConfig {
18
+ targetId: RingIdRaw
19
+ parallelism: number
20
+ noProgressLimit: number
21
+ peerManager: PeerManager
22
+ // Note that contacted peers will be mutated by the DiscoverySession or other parallel sessions
23
+ contactedPeers: Set<DhtAddress>
24
+ }
25
+
26
+ export class RingDiscoverySession {
27
+
28
+ public readonly id = v4()
29
+ private stopped = false
30
+ private emitter = new EventEmitter<RingDiscoverySessionEvents>()
31
+ private noProgressCounter = 0
32
+ private ongoingClosestPeersRequests: Set<DhtAddress> = new Set()
33
+ private readonly config: RingDiscoverySessionConfig
34
+ private numContactedPeers = 0
35
+ private targetIdAsRingId: RingId
36
+
37
+ constructor(config: RingDiscoverySessionConfig) {
38
+ this.config = config
39
+ this.targetIdAsRingId = getRingIdFromRaw(this.config.targetId)
40
+ }
41
+
42
+ private addContacts(contacts: PeerDescriptor[]): void {
43
+ if (this.stopped) {
44
+ return
45
+ }
46
+ this.config.peerManager.addContact(contacts)
47
+ }
48
+
49
+ private async getClosestPeersFromContact(contact: DhtNodeRpcRemote): Promise<RingContacts> {
50
+ if (this.stopped) {
51
+ return { left: [], right: [] }
52
+ }
53
+ logger.trace(`Getting closest ring peers from contact: ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
54
+ this.numContactedPeers++
55
+ this.config.contactedPeers.add(contact.getNodeId())
56
+ const returnedContacts = await contact.getClosestRingPeers(this.config.targetId)
57
+ this.config.peerManager.setContactActive(contact.getNodeId())
58
+ return returnedContacts
59
+ }
60
+
61
+ private onClosestPeersRequestSucceeded(nodeId: DhtAddress, contacts: RingContacts) {
62
+ if (!this.ongoingClosestPeersRequests.has(nodeId)) {
63
+ return
64
+ }
65
+ this.ongoingClosestPeersRequests.delete(nodeId)
66
+ const oldClosestContacts = this.config.peerManager.getClosestRingContactsTo(this.config.targetId, 1)
67
+ const oldClosestLeftDistance = getLeftDistance(
68
+ this.targetIdAsRingId,
69
+ getRingIdFromPeerDescriptor(oldClosestContacts.left[0].getPeerDescriptor())
70
+ )
71
+ const oldClosestRightDistance = getLeftDistance(
72
+ this.targetIdAsRingId,
73
+ getRingIdFromPeerDescriptor(oldClosestContacts.right[0].getPeerDescriptor())
74
+ )
75
+ this.addContacts(contacts.left.concat(contacts.right))
76
+ const newClosestContacts = this.config.peerManager.getClosestRingContactsTo(this.config.targetId, 1)
77
+ const newClosestLeftDistance = getLeftDistance(this.targetIdAsRingId,
78
+ getRingIdFromPeerDescriptor(newClosestContacts.left[0].getPeerDescriptor()))
79
+ const newClosestRightDistance = getLeftDistance(this.targetIdAsRingId,
80
+ getRingIdFromPeerDescriptor(newClosestContacts.right[0].getPeerDescriptor()))
81
+ if (newClosestLeftDistance >= oldClosestLeftDistance && newClosestRightDistance >= oldClosestRightDistance) {
82
+ this.noProgressCounter++
83
+ }
84
+ }
85
+
86
+ private onClosestPeersRequestFailed(peer: DhtNodeRpcRemote) {
87
+ if (!this.ongoingClosestPeersRequests.has(peer.getNodeId())) {
88
+ return
89
+ }
90
+ this.ongoingClosestPeersRequests.delete(peer.getNodeId())
91
+ this.config.peerManager.removeContact(peer.getNodeId())
92
+ }
93
+
94
+ private findMoreContacts(): void {
95
+ if (this.stopped) {
96
+ return
97
+ }
98
+ const uncontacted = this.config.peerManager.getClosestRingContactsTo(
99
+ this.config.targetId,
100
+ this.config.parallelism,
101
+ this.config.contactedPeers
102
+ )
103
+ if ((uncontacted.left.length === 0 && uncontacted.right.length === 0)
104
+ || this.noProgressCounter >= this.config.noProgressLimit) {
105
+ this.emitter.emit('discoveryCompleted')
106
+ this.stopped = true
107
+ return
108
+ }
109
+ // ask from both sides equally
110
+ const merged = []
111
+ const alreadyInMerged: Set<DhtAddress> = new Set()
112
+ const length = Math.max(uncontacted.left.length, uncontacted.right.length)
113
+ for (let i = 0; i < length; i++) {
114
+ if (i < uncontacted.left.length) {
115
+ if (!alreadyInMerged.has(uncontacted.left[i].getNodeId())) {
116
+ merged.push(uncontacted.left[i])
117
+ alreadyInMerged.add(uncontacted.left[i].getNodeId())
118
+ }
119
+ }
120
+ if (i < uncontacted.right.length) {
121
+ if (!alreadyInMerged.has(uncontacted.right[i].getNodeId())) {
122
+ merged.push(uncontacted.right[i])
123
+ alreadyInMerged.add(uncontacted.right[i].getNodeId())
124
+ }
125
+ }
126
+ }
127
+
128
+ for (const nextPeer of merged) {
129
+ if (this.ongoingClosestPeersRequests.size >= this.config.parallelism) {
130
+ break
131
+ }
132
+ this.ongoingClosestPeersRequests.add(nextPeer.getNodeId())
133
+ // eslint-disable-next-line promise/catch-or-return
134
+ this.getClosestPeersFromContact(nextPeer)
135
+ .then((contacts) => this.onClosestPeersRequestSucceeded(nextPeer.getNodeId(), contacts))
136
+ .catch(() => this.onClosestPeersRequestFailed(nextPeer))
137
+ .finally(() => {
138
+ this.findMoreContacts()
139
+ })
140
+ }
141
+ }
142
+
143
+ public async findClosestNodes(timeout: number): Promise<void> {
144
+ if (this.config.peerManager.getContactCount(this.config.contactedPeers) === 0) {
145
+ return
146
+ }
147
+ // TODO add abortController and signal it in stop()
148
+ await runAndWaitForEvents3<RingDiscoverySessionEvents>(
149
+ [this.findMoreContacts.bind(this)],
150
+ [[this.emitter, 'discoveryCompleted']],
151
+ timeout
152
+ )
153
+ }
154
+
155
+ public stop(): void {
156
+ this.stopped = true
157
+ this.emitter.emit('discoveryCompleted')
158
+ this.emitter.removeAllListeners()
159
+ }
160
+ }
@@ -8,8 +8,7 @@ import {
8
8
  RouteMessageWrapper,
9
9
  RouteMessageAck,
10
10
  RecursiveOperationRequest,
11
- Message,
12
- MessageType
11
+ Message
13
12
  } from '../../proto/packages/dht/protos/DhtRpc'
14
13
  import { ITransport } from '../../transport/ITransport'
15
14
  import { ListeningRpcCommunicator } from '../../transport/ListeningRpcCommunicator'
@@ -90,7 +89,6 @@ export class RecursiveOperationSession extends EventEmitter<RecursiveOperationSe
90
89
  operation: this.config.operation
91
90
  }
92
91
  const msg: Message = {
93
- messageType: MessageType.RECURSIVE_OPERATION_REQUEST,
94
92
  messageId: v4(),
95
93
  serviceId,
96
94
  body: {
@@ -98,7 +98,7 @@ export class StoreManager {
98
98
 
99
99
  public async storeDataToDht(key: DhtAddress, data: Any, creator: DhtAddress): Promise<PeerDescriptor[]> {
100
100
  logger.debug(`Storing data to DHT ${this.config.serviceId}`)
101
- const result = await this.config.recursiveOperationManager.execute(key, RecursiveOperation.FIND_NODE)
101
+ const result = await this.config.recursiveOperationManager.execute(key, RecursiveOperation.FIND_CLOSEST_NODES)
102
102
  const closestNodes = result.closestNodes
103
103
  const successfulNodes: PeerDescriptor[] = []
104
104
  const ttl = this.config.highestTtl // ToDo: make TTL decrease according to some nice curve
package/src/exports.ts CHANGED
@@ -17,6 +17,7 @@ export { ClientWebsocket } from './connection/websocket/ClientWebsocket'
17
17
  export { ManagedConnection } from './connection/ManagedConnection'
18
18
  export { ConnectionType } from './connection/IConnection'
19
19
  export { ServiceID } from './types/ServiceID'
20
+ export { RingContacts } from './dht/contact/RingContactList'
20
21
  export {
21
22
  DhtAddress,
22
23
  DhtAddressRaw,
@@ -29,7 +29,7 @@ const calculateNodeIdRaw = (ipAddress: number, privateKey: Uint8Array): DhtAddre
29
29
  return nodeIdRaw
30
30
  }
31
31
 
32
- export const createPeerDescriptor = (connectivityResponse: ConnectivityResponse, nodeId?: DhtAddress): PeerDescriptor => {
32
+ export const createPeerDescriptor = (connectivityResponse: ConnectivityResponse, region: number, nodeId?: DhtAddress): PeerDescriptor => {
33
33
  const privateKey = crypto.randomBytes(32)
34
34
  const publicKey = crypto.randomBytes(20) // TODO calculate publicKey from privateKey
35
35
  let nodeIdRaw: DhtAddressRaw
@@ -42,6 +42,7 @@ export const createPeerDescriptor = (connectivityResponse: ConnectivityResponse,
42
42
  nodeId: nodeIdRaw,
43
43
  type: isBrowserEnvironment() ? NodeType.BROWSER : NodeType.NODEJS,
44
44
  ipAddress: connectivityResponse.ipAddress,
45
+ region,
45
46
  publicKey
46
47
  }
47
48
  if (connectivityResponse.websocket) {
@@ -0,0 +1,32 @@
1
+ export const LOCAL_PROTOCOL_VERSION = '1.0'
2
+
3
+ /*
4
+ * When two nodes negotiate whether they are compatible or not, it is up to the
5
+ * newer version to decide if it supports the old version or not.
6
+ *
7
+ * The older version assumes optimistically that it may be supported by the newer
8
+ * version. It can't know for sure, but the other node will tell if it is not
9
+ * supported (e.g. rejecting the handshake with UNSUPPORTED_VERSION error).
10
+ */
11
+ export const isMaybeSupportedVersion = (remoteVersion: string): boolean => {
12
+ const localMajor = parseVersion(LOCAL_PROTOCOL_VERSION)!.major
13
+ const remoteMajor = parseVersion(remoteVersion)?.major
14
+ if ((remoteMajor === undefined) || (remoteMajor < localMajor)) {
15
+ return false
16
+ } else {
17
+ // TODO implement proper checking when there are new protocol versions
18
+ return true
19
+ }
20
+ }
21
+
22
+ export const parseVersion = (version: string): { major: number, minor: number } | undefined => {
23
+ const parts = version.split('.')
24
+ if (parts.length === 2) {
25
+ const values = parts.map((p) => Number(p))
26
+ if (!values.some((v) => isNaN(v))) {
27
+ return { major: values[0], minor: values[1] }
28
+ }
29
+ } else {
30
+ return undefined
31
+ }
32
+ }
@@ -4,8 +4,8 @@
4
4
  import { ExternalApiRpc } from "./DhtRpc";
5
5
  import type { ExternalStoreDataResponse } from "./DhtRpc";
6
6
  import type { ExternalStoreDataRequest } from "./DhtRpc";
7
- import type { ExternalFindDataResponse } from "./DhtRpc";
8
- import type { ExternalFindDataRequest } from "./DhtRpc";
7
+ import type { ExternalFetchDataResponse } from "./DhtRpc";
8
+ import type { ExternalFetchDataRequest } from "./DhtRpc";
9
9
  import { ConnectionLockRpc } from "./DhtRpc";
10
10
  import type { DisconnectNotice } from "./DhtRpc";
11
11
  import type { UnlockRequest } from "./DhtRpc";
@@ -35,6 +35,8 @@ import type { Empty } from "../../../google/protobuf/empty";
35
35
  import type { LeaveNotice } from "./DhtRpc";
36
36
  import type { PingResponse } from "./DhtRpc";
37
37
  import type { PingRequest } from "./DhtRpc";
38
+ import type { ClosestRingPeersResponse } from "./DhtRpc";
39
+ import type { ClosestRingPeersRequest } from "./DhtRpc";
38
40
  import { stackIntercept } from "@protobuf-ts/runtime-rpc";
39
41
  import type { ClosestPeersResponse } from "./DhtRpc";
40
42
  import type { ClosestPeersRequest } from "./DhtRpc";
@@ -48,6 +50,10 @@ export interface IDhtNodeRpcClient {
48
50
  * @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
49
51
  */
50
52
  getClosestPeers(input: ClosestPeersRequest, options?: RpcOptions): UnaryCall<ClosestPeersRequest, ClosestPeersResponse>;
53
+ /**
54
+ * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
55
+ */
56
+ getClosestRingPeers(input: ClosestRingPeersRequest, options?: RpcOptions): UnaryCall<ClosestRingPeersRequest, ClosestRingPeersResponse>;
51
57
  /**
52
58
  * @generated from protobuf rpc: ping(dht.PingRequest) returns (dht.PingResponse);
53
59
  */
@@ -73,18 +79,25 @@ export class DhtNodeRpcClient implements IDhtNodeRpcClient, ServiceInfo {
73
79
  const method = this.methods[0], opt = this._transport.mergeOptions(options);
74
80
  return stackIntercept<ClosestPeersRequest, ClosestPeersResponse>("unary", this._transport, method, opt, input);
75
81
  }
82
+ /**
83
+ * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
84
+ */
85
+ getClosestRingPeers(input: ClosestRingPeersRequest, options?: RpcOptions): UnaryCall<ClosestRingPeersRequest, ClosestRingPeersResponse> {
86
+ const method = this.methods[1], opt = this._transport.mergeOptions(options);
87
+ return stackIntercept<ClosestRingPeersRequest, ClosestRingPeersResponse>("unary", this._transport, method, opt, input);
88
+ }
76
89
  /**
77
90
  * @generated from protobuf rpc: ping(dht.PingRequest) returns (dht.PingResponse);
78
91
  */
79
92
  ping(input: PingRequest, options?: RpcOptions): UnaryCall<PingRequest, PingResponse> {
80
- const method = this.methods[1], opt = this._transport.mergeOptions(options);
93
+ const method = this.methods[2], opt = this._transport.mergeOptions(options);
81
94
  return stackIntercept<PingRequest, PingResponse>("unary", this._transport, method, opt, input);
82
95
  }
83
96
  /**
84
97
  * @generated from protobuf rpc: leaveNotice(dht.LeaveNotice) returns (google.protobuf.Empty);
85
98
  */
86
99
  leaveNotice(input: LeaveNotice, options?: RpcOptions): UnaryCall<LeaveNotice, Empty> {
87
- const method = this.methods[2], opt = this._transport.mergeOptions(options);
100
+ const method = this.methods[3], opt = this._transport.mergeOptions(options);
88
101
  return stackIntercept<LeaveNotice, Empty>("unary", this._transport, method, opt, input);
89
102
  }
90
103
  }
@@ -352,9 +365,9 @@ export class ConnectionLockRpcClient implements IConnectionLockRpcClient, Servic
352
365
  */
353
366
  export interface IExternalApiRpcClient {
354
367
  /**
355
- * @generated from protobuf rpc: externalFindData(dht.ExternalFindDataRequest) returns (dht.ExternalFindDataResponse);
368
+ * @generated from protobuf rpc: externalFetchData(dht.ExternalFetchDataRequest) returns (dht.ExternalFetchDataResponse);
356
369
  */
357
- externalFindData(input: ExternalFindDataRequest, options?: RpcOptions): UnaryCall<ExternalFindDataRequest, ExternalFindDataResponse>;
370
+ externalFetchData(input: ExternalFetchDataRequest, options?: RpcOptions): UnaryCall<ExternalFetchDataRequest, ExternalFetchDataResponse>;
358
371
  /**
359
372
  * @generated from protobuf rpc: externalStoreData(dht.ExternalStoreDataRequest) returns (dht.ExternalStoreDataResponse);
360
373
  */
@@ -370,11 +383,11 @@ export class ExternalApiRpcClient implements IExternalApiRpcClient, ServiceInfo
370
383
  constructor(private readonly _transport: RpcTransport) {
371
384
  }
372
385
  /**
373
- * @generated from protobuf rpc: externalFindData(dht.ExternalFindDataRequest) returns (dht.ExternalFindDataResponse);
386
+ * @generated from protobuf rpc: externalFetchData(dht.ExternalFetchDataRequest) returns (dht.ExternalFetchDataResponse);
374
387
  */
375
- externalFindData(input: ExternalFindDataRequest, options?: RpcOptions): UnaryCall<ExternalFindDataRequest, ExternalFindDataResponse> {
388
+ externalFetchData(input: ExternalFetchDataRequest, options?: RpcOptions): UnaryCall<ExternalFetchDataRequest, ExternalFetchDataResponse> {
376
389
  const method = this.methods[0], opt = this._transport.mergeOptions(options);
377
- return stackIntercept<ExternalFindDataRequest, ExternalFindDataResponse>("unary", this._transport, method, opt, input);
390
+ return stackIntercept<ExternalFetchDataRequest, ExternalFetchDataResponse>("unary", this._transport, method, opt, input);
378
391
  }
379
392
  /**
380
393
  * @generated from protobuf rpc: externalStoreData(dht.ExternalStoreDataRequest) returns (dht.ExternalStoreDataResponse);
@@ -3,8 +3,8 @@
3
3
  // tslint:disable
4
4
  import { ExternalStoreDataResponse } from "./DhtRpc";
5
5
  import { ExternalStoreDataRequest } from "./DhtRpc";
6
- import { ExternalFindDataResponse } from "./DhtRpc";
7
- import { ExternalFindDataRequest } from "./DhtRpc";
6
+ import { ExternalFetchDataResponse } from "./DhtRpc";
7
+ import { ExternalFetchDataRequest } from "./DhtRpc";
8
8
  import { DisconnectNotice } from "./DhtRpc";
9
9
  import { UnlockRequest } from "./DhtRpc";
10
10
  import { LockResponse } from "./DhtRpc";
@@ -24,6 +24,8 @@ import { Empty } from "../../../google/protobuf/empty";
24
24
  import { LeaveNotice } from "./DhtRpc";
25
25
  import { PingResponse } from "./DhtRpc";
26
26
  import { PingRequest } from "./DhtRpc";
27
+ import { ClosestRingPeersResponse } from "./DhtRpc";
28
+ import { ClosestRingPeersRequest } from "./DhtRpc";
27
29
  import { ClosestPeersResponse } from "./DhtRpc";
28
30
  import { ClosestPeersRequest } from "./DhtRpc";
29
31
  import { ServerCallContext } from "@protobuf-ts/runtime-rpc";
@@ -35,6 +37,10 @@ export interface IDhtNodeRpc<T = ServerCallContext> {
35
37
  * @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
36
38
  */
37
39
  getClosestPeers(request: ClosestPeersRequest, context: T): Promise<ClosestPeersResponse>;
40
+ /**
41
+ * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
42
+ */
43
+ getClosestRingPeers(request: ClosestRingPeersRequest, context: T): Promise<ClosestRingPeersResponse>;
38
44
  /**
39
45
  * @generated from protobuf rpc: ping(dht.PingRequest) returns (dht.PingResponse);
40
46
  */
@@ -140,9 +146,9 @@ export interface IConnectionLockRpc<T = ServerCallContext> {
140
146
  */
141
147
  export interface IExternalApiRpc<T = ServerCallContext> {
142
148
  /**
143
- * @generated from protobuf rpc: externalFindData(dht.ExternalFindDataRequest) returns (dht.ExternalFindDataResponse);
149
+ * @generated from protobuf rpc: externalFetchData(dht.ExternalFetchDataRequest) returns (dht.ExternalFetchDataResponse);
144
150
  */
145
- externalFindData(request: ExternalFindDataRequest, context: T): Promise<ExternalFindDataResponse>;
151
+ externalFetchData(request: ExternalFetchDataRequest, context: T): Promise<ExternalFetchDataResponse>;
146
152
  /**
147
153
  * @generated from protobuf rpc: externalStoreData(dht.ExternalStoreDataRequest) returns (dht.ExternalStoreDataResponse);
148
154
  */