@streamr/trackerless-network 100.0.0-testnet-three.6 → 100.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/package.json +7 -7
  2. package/dist/src/NetworkStack.d.ts +3 -3
  3. package/dist/src/NetworkStack.js +6 -6
  4. package/dist/src/NetworkStack.js.map +1 -1
  5. package/dist/src/exports.d.ts +2 -2
  6. package/dist/src/exports.js +7 -1
  7. package/dist/src/exports.js.map +1 -1
  8. package/dist/src/logic/EntryPointDiscovery.d.ts +1 -1
  9. package/dist/src/logic/EntryPointDiscovery.js +2 -2
  10. package/dist/src/logic/EntryPointDiscovery.js.map +1 -1
  11. package/dist/src/logic/Layer0Node.d.ts +1 -1
  12. package/dist/src/logic/Layer1Node.d.ts +7 -1
  13. package/dist/src/logic/NodeList.d.ts +2 -1
  14. package/dist/src/logic/NodeList.js +7 -2
  15. package/dist/src/logic/NodeList.js.map +1 -1
  16. package/dist/src/logic/RandomGraphNode.d.ts +3 -0
  17. package/dist/src/logic/RandomGraphNode.js +26 -4
  18. package/dist/src/logic/RandomGraphNode.js.map +1 -1
  19. package/dist/src/logic/StreamrNode.js +9 -4
  20. package/dist/src/logic/StreamrNode.js.map +1 -1
  21. package/dist/src/logic/createRandomGraphNode.d.ts +1 -1
  22. package/dist/src/logic/createRandomGraphNode.js +15 -6
  23. package/dist/src/logic/createRandomGraphNode.js.map +1 -1
  24. package/dist/src/logic/inspect/Inspector.js +2 -2
  25. package/dist/src/logic/inspect/Inspector.js.map +1 -1
  26. package/dist/src/logic/neighbor-discovery/HandshakeRpcLocal.d.ts +1 -2
  27. package/dist/src/logic/neighbor-discovery/HandshakeRpcLocal.js +3 -8
  28. package/dist/src/logic/neighbor-discovery/HandshakeRpcLocal.js.map +1 -1
  29. package/dist/src/logic/neighbor-discovery/Handshaker.d.ts +3 -2
  30. package/dist/src/logic/neighbor-discovery/Handshaker.js +29 -10
  31. package/dist/src/logic/neighbor-discovery/Handshaker.js.map +1 -1
  32. package/dist/src/logic/neighbor-discovery/NeighborFinder.d.ts +3 -0
  33. package/dist/src/logic/neighbor-discovery/NeighborFinder.js +7 -1
  34. package/dist/src/logic/neighbor-discovery/NeighborFinder.js.map +1 -1
  35. package/dist/src/logic/neighbor-discovery/NeighborUpdateManager.d.ts +1 -2
  36. package/dist/src/logic/neighbor-discovery/NeighborUpdateManager.js +0 -1
  37. package/dist/src/logic/neighbor-discovery/NeighborUpdateManager.js.map +1 -1
  38. package/dist/src/logic/neighbor-discovery/NeighborUpdateRpcLocal.d.ts +1 -2
  39. package/dist/src/logic/neighbor-discovery/NeighborUpdateRpcLocal.js +0 -1
  40. package/dist/src/logic/neighbor-discovery/NeighborUpdateRpcLocal.js.map +1 -1
  41. package/dist/src/logic/protocol-integration/stream-message/GroupKeyResponseTranslator.js.map +1 -1
  42. package/dist/src/logic/protocol-integration/stream-message/StreamMessageTranslator.js +58 -52
  43. package/dist/src/logic/protocol-integration/stream-message/StreamMessageTranslator.js.map +1 -1
  44. package/dist/src/logic/protocol-integration/stream-message/oldStreamMessageBinaryUtils.d.ts +5 -1
  45. package/dist/src/logic/protocol-integration/stream-message/oldStreamMessageBinaryUtils.js +19 -1
  46. package/dist/src/logic/protocol-integration/stream-message/oldStreamMessageBinaryUtils.js.map +1 -1
  47. package/dist/src/logic/proxy/ProxyConnectionRpcLocal.js +2 -2
  48. package/dist/src/logic/proxy/ProxyConnectionRpcLocal.js.map +1 -1
  49. package/dist/src/logic/temporary-connection/TemporaryConnectionRpcLocal.d.ts +5 -1
  50. package/dist/src/logic/temporary-connection/TemporaryConnectionRpcLocal.js +5 -0
  51. package/dist/src/logic/temporary-connection/TemporaryConnectionRpcLocal.js.map +1 -1
  52. package/dist/src/proto/packages/dht/protos/DhtRpc.client.d.ts +6 -6
  53. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js +2 -2
  54. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js.map +1 -1
  55. package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +54 -87
  56. package/dist/src/proto/packages/dht/protos/DhtRpc.js +17 -49
  57. package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
  58. package/dist/src/proto/packages/dht/protos/DhtRpc.server.d.ts +4 -4
  59. package/dist/src/proto/packages/trackerless-network/protos/NetworkRpc.d.ts +53 -21
  60. package/dist/src/proto/packages/trackerless-network/protos/NetworkRpc.js +24 -11
  61. package/dist/src/proto/packages/trackerless-network/protos/NetworkRpc.js.map +1 -1
  62. package/dist/test/benchmark/first-message.js +1 -1
  63. package/dist/test/benchmark/first-message.js.map +1 -1
  64. package/dist/test/utils/utils.js +9 -5
  65. package/dist/test/utils/utils.js.map +1 -1
  66. package/package.json +7 -7
  67. package/protos/NetworkRpc.proto +16 -9
  68. package/src/NetworkStack.ts +9 -9
  69. package/src/exports.ts +6 -2
  70. package/src/logic/EntryPointDiscovery.ts +3 -3
  71. package/src/logic/Layer0Node.ts +1 -1
  72. package/src/logic/Layer1Node.ts +16 -1
  73. package/src/logic/NodeList.ts +9 -3
  74. package/src/logic/RandomGraphNode.ts +74 -10
  75. package/src/logic/StreamrNode.ts +9 -4
  76. package/src/logic/createRandomGraphNode.ts +16 -7
  77. package/src/logic/inspect/Inspector.ts +2 -2
  78. package/src/logic/neighbor-discovery/HandshakeRpcLocal.ts +5 -10
  79. package/src/logic/neighbor-discovery/Handshaker.ts +32 -12
  80. package/src/logic/neighbor-discovery/NeighborFinder.ts +10 -1
  81. package/src/logic/neighbor-discovery/NeighborUpdateManager.ts +1 -3
  82. package/src/logic/neighbor-discovery/NeighborUpdateRpcLocal.ts +1 -3
  83. package/src/logic/protocol-integration/stream-message/GroupKeyResponseTranslator.ts +0 -1
  84. package/src/logic/protocol-integration/stream-message/StreamMessageTranslator.ts +59 -62
  85. package/src/logic/protocol-integration/stream-message/oldStreamMessageBinaryUtils.ts +28 -2
  86. package/src/logic/proxy/ProxyConnectionRpcLocal.ts +3 -5
  87. package/src/logic/temporary-connection/TemporaryConnectionRpcLocal.ts +10 -2
  88. package/src/proto/packages/dht/protos/DhtRpc.client.ts +7 -7
  89. package/src/proto/packages/dht/protos/DhtRpc.server.ts +4 -4
  90. package/src/proto/packages/dht/protos/DhtRpc.ts +64 -100
  91. package/src/proto/packages/trackerless-network/protos/NetworkRpc.ts +69 -31
  92. package/test/benchmark/StreamPartIdDataKeyDistribution.test.ts +60 -0
  93. package/test/benchmark/first-message.ts +1 -1
  94. package/test/end-to-end/proxy-key-exchange.test.ts +13 -10
  95. package/test/integration/Handshakes.test.ts +7 -3
  96. package/test/integration/stream-without-default-entrypoints.test.ts +1 -1
  97. package/test/integration/streamEntryPointReplacing.test.ts +5 -5
  98. package/test/unit/EntrypointDiscovery.test.ts +4 -4
  99. package/test/unit/HandshakeRpcLocal.test.ts +18 -2
  100. package/test/unit/Handshaker.test.ts +8 -3
  101. package/test/unit/NeighborFinder.test.ts +3 -0
  102. package/test/unit/NeighborUpdateRpcLocal.test.ts +0 -4
  103. package/test/unit/Propagation.test.ts +10 -7
  104. package/test/unit/StreamMessageTranslator.test.ts +3 -4
  105. package/test/unit/StreamPartIDDataKey.test.ts +12 -0
  106. package/test/unit/TemporaryConnectionRpcLocal.test.ts +7 -1
  107. package/test/utils/mock/MockLayer0Node.ts +1 -1
  108. package/test/utils/mock/MockLayer1Node.ts +3 -0
  109. package/test/utils/utils.ts +10 -7
  110. package/test/unit/GroupKeyRequestTranslator.test.ts +0 -36
@@ -1,4 +1,4 @@
1
- import {
1
+ import {
2
2
  ConnectionManager,
3
3
  DhtNode,
4
4
  DhtNodeOptions,
@@ -6,15 +6,15 @@ import {
6
6
  PeerDescriptor,
7
7
  areEqualPeerDescriptors
8
8
  } from '@streamr/dht'
9
- import { StreamrNode, StreamrNodeConfig } from './logic/StreamrNode'
10
- import { Logger, MetricsContext, waitForCondition } from '@streamr/utils'
11
9
  import { StreamID, StreamPartID, toStreamPartID } from '@streamr/protocol'
12
- import { NodeInfoResponse, ProxyDirection, StreamMessage, StreamMessageType } from './proto/packages/trackerless-network/protos/NetworkRpc'
13
- import { Layer0Node } from './logic/Layer0Node'
10
+ import { Logger, MetricsContext, waitForCondition } from '@streamr/utils'
14
11
  import { pull } from 'lodash'
15
- import { NODE_INFO_RPC_SERVICE_ID, NodeInfoRpcLocal } from './logic/node-info/NodeInfoRpcLocal'
12
+ import { version as applicationVersion } from '../package.json'
13
+ import { Layer0Node } from './logic/Layer0Node'
14
+ import { StreamrNode, StreamrNodeConfig } from './logic/StreamrNode'
16
15
  import { NodeInfoClient } from './logic/node-info/NodeInfoClient'
17
- import { version as localVersion } from '../package.json'
16
+ import { NODE_INFO_RPC_SERVICE_ID, NodeInfoRpcLocal } from './logic/node-info/NodeInfoRpcLocal'
17
+ import { NodeInfoResponse, ProxyDirection, StreamMessage } from './proto/packages/trackerless-network/protos/NetworkRpc'
18
18
 
19
19
  export interface NetworkOptions {
20
20
  layer0?: DhtNodeOptions
@@ -90,7 +90,7 @@ export class NetworkStack {
90
90
 
91
91
  async broadcast(msg: StreamMessage): Promise<void> {
92
92
  const streamPartId = toStreamPartID(msg.messageId!.streamId as StreamID, msg.messageId!.streamPartition)
93
- if (this.getStreamrNode().isProxiedStreamPart(streamPartId, ProxyDirection.SUBSCRIBE) && (msg.messageType === StreamMessageType.MESSAGE)) {
93
+ if (this.getStreamrNode().isProxiedStreamPart(streamPartId, ProxyDirection.SUBSCRIBE) && (msg.body.oneofKind === 'contentMessage')) {
94
94
  throw new Error(`Cannot broadcast to ${streamPartId} as proxy subscribe connections have been set`)
95
95
  }
96
96
  // TODO could combine these two calls to isProxiedStreamPart?
@@ -168,7 +168,7 @@ export class NetworkStack {
168
168
  neighbors: this.getLayer0Node().getNeighbors()
169
169
  },
170
170
  streamPartitions: this.getStreamrNode().getNodeInfo(),
171
- version: localVersion
171
+ version: applicationVersion
172
172
  }
173
173
  }
174
174
 
package/src/exports.ts CHANGED
@@ -1,9 +1,13 @@
1
1
  export { NetworkStack, NetworkOptions, NodeInfo } from './NetworkStack'
2
2
  export { NetworkNode, createNetworkNode } from './NetworkNode'
3
3
  export { StreamrNodeConfig } from './logic/StreamrNode'
4
- export { ProxyDirection } from './proto/packages/trackerless-network/protos/NetworkRpc'
4
+ export { ProxyDirection, GroupKeyRequest, GroupKeyResponse } from './proto/packages/trackerless-network/protos/NetworkRpc'
5
5
  export { streamPartIdToDataKey } from './logic/EntryPointDiscovery'
6
6
  export {
7
7
  convertStreamMessageToBytes,
8
- convertBytesToStreamMessage
8
+ convertBytesToStreamMessage,
9
+ convertGroupKeyRequestToBytes,
10
+ convertBytesToGroupKeyRequest,
11
+ convertGroupKeyResponseToBytes,
12
+ convertBytesToGroupKeyResponse
9
13
  } from './logic/protocol-integration/stream-message/oldStreamMessageBinaryUtils'
@@ -13,7 +13,7 @@ import { Any } from '../proto/google/protobuf/any'
13
13
  import { Layer1Node } from './Layer1Node'
14
14
 
15
15
  export const streamPartIdToDataKey = (streamPartId: StreamPartID): DhtAddress => {
16
- return getDhtAddressFromRaw(new Uint8Array(createHash('md5').update(streamPartId).digest()))
16
+ return getDhtAddressFromRaw(new Uint8Array((createHash('sha1').update(streamPartId).digest())))
17
17
  }
18
18
 
19
19
  const parseEntryPointData = (dataEntries: DataEntry[]): PeerDescriptor[] => {
@@ -60,7 +60,7 @@ interface EntryPointDiscoveryConfig {
60
60
  streamPartId: StreamPartID
61
61
  localPeerDescriptor: PeerDescriptor
62
62
  layer1Node: Layer1Node
63
- getEntryPointData: (key: DhtAddress) => Promise<DataEntry[]>
63
+ fetchEntryPointData: (key: DhtAddress) => Promise<DataEntry[]>
64
64
  storeEntryPointData: (key: DhtAddress, data: Any) => Promise<PeerDescriptor[]>
65
65
  deleteEntryPointData: (key: DhtAddress) => Promise<void>
66
66
  storeInterval?: number
@@ -111,7 +111,7 @@ export class EntryPointDiscovery {
111
111
  private async queryEntrypoints(key: DhtAddress): Promise<PeerDescriptor[]> {
112
112
  logger.trace(`Finding data from dht node ${getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)}`)
113
113
  try {
114
- const result = await this.config.getEntryPointData(key)
114
+ const result = await this.config.fetchEntryPointData(key)
115
115
  return parseEntryPointData(result)
116
116
  } catch (err) {
117
117
  return []
@@ -5,7 +5,7 @@ export interface Layer0Node extends ITransport {
5
5
  joinDht(entryPointDescriptors: PeerDescriptor[]): Promise<void>
6
6
  hasJoined(): boolean
7
7
  getLocalPeerDescriptor(): PeerDescriptor
8
- getDataFromDht(key: DhtAddress): Promise<DataEntry[]>
8
+ fetchDataFromDht(key: DhtAddress): Promise<DataEntry[]>
9
9
  storeDataToDht(key: DhtAddress, data: Any): Promise<PeerDescriptor[]>
10
10
  deleteDataFromDht(key: DhtAddress, waitForCompletion: boolean): Promise<void>
11
11
  waitForNetworkConnectivity(): Promise<void>
@@ -1,21 +1,36 @@
1
- import { DhtAddress, PeerDescriptor } from '@streamr/dht'
1
+ import { DhtAddress, PeerDescriptor, RingContacts } from '@streamr/dht'
2
2
 
3
3
  export interface Layer1NodeEvents {
4
4
  contactAdded: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
5
5
  contactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
6
6
  randomContactAdded: (peerDescriptor: PeerDescriptor, randomPeers: PeerDescriptor[]) => void
7
7
  randomContactRemoved: (peerDescriptor: PeerDescriptor, randomPeers: PeerDescriptor[]) => void
8
+ ringContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
9
+ ringContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
8
10
  }
9
11
 
10
12
  export interface Layer1Node {
11
13
  on<T extends keyof Layer1NodeEvents>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
12
14
  once<T extends keyof Layer1NodeEvents>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
13
15
  off<T extends keyof Layer1NodeEvents>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
16
+ on<T extends keyof Layer1NodeEvents>(
17
+ eventName: T,
18
+ listener: (peerDescriptor: PeerDescriptor, peers: RingContacts) => void
19
+ ): void
20
+ once<T extends keyof Layer1NodeEvents>(
21
+ eventName: T,
22
+ listener: (peerDescriptor: PeerDescriptor, peers: RingContacts) => void
23
+ ): void
24
+ off<T extends keyof Layer1NodeEvents>(
25
+ eventName: T,
26
+ listener: (peerDescriptor: PeerDescriptor, peers: RingContacts
27
+ ) => void): void
14
28
  removeContact: (nodeId: DhtAddress) => void
15
29
  getClosestContacts: (maxCount?: number) => PeerDescriptor[]
16
30
  getNeighbors: () => PeerDescriptor[]
17
31
  getNeighborCount(): number
18
32
  joinDht: (entryPoints: PeerDescriptor[], doRandomJoin?: boolean, retry?: boolean) => Promise<void>
33
+ joinRing: () => Promise<void>
19
34
  start: () => Promise<void>
20
35
  stop: () => Promise<void>
21
36
  }
@@ -4,7 +4,8 @@ import { DeliveryRpcRemote } from './DeliveryRpcRemote'
4
4
  import { EventEmitter } from 'eventemitter3'
5
5
 
6
6
  export interface Events {
7
- nodeAdded: (id: DhtAddress, remote: DeliveryRpcRemote) => any
7
+ nodeAdded: (id: DhtAddress, remote: DeliveryRpcRemote) => void
8
+ nodeRemoved: (id: DhtAddress, remote: DeliveryRpcRemote) => void
8
9
  }
9
10
 
10
11
  const getValuesOfIncludedKeys = (nodes: Map<DhtAddress, DeliveryRpcRemote>, exclude: DhtAddress[]): DeliveryRpcRemote[] => {
@@ -40,13 +41,18 @@ export class NodeList extends EventEmitter<Events> {
40
41
  }
41
42
 
42
43
  remove(nodeId: DhtAddress): void {
43
- this.nodes.delete(nodeId)
44
+ if (this.nodes.has(nodeId)) {
45
+ const remote = this.nodes.get(nodeId)!
46
+ this.nodes.delete(nodeId)
47
+ this.emit('nodeRemoved', nodeId, remote)
48
+ }
44
49
  }
45
50
 
46
51
  has(nodeId: DhtAddress): boolean {
47
52
  return this.nodes.has(nodeId)
48
53
  }
49
54
 
55
+ // Replace nodes does not emit nodeRemoved events, use with caution
50
56
  replaceAll(neighbors: DeliveryRpcRemote[]): void {
51
57
  this.nodes.clear()
52
58
  const limited = neighbors.splice(0, this.limit)
@@ -94,7 +100,7 @@ export class NodeList extends EventEmitter<Events> {
94
100
  }
95
101
 
96
102
  stop(): void {
97
- this.nodes.clear()
103
+ this.nodes.forEach((node) => this.remove(getNodeIdFromPeerDescriptor(node.getPeerDescriptor())))
98
104
  this.removeAllListeners()
99
105
  }
100
106
  }
@@ -5,7 +5,8 @@ import {
5
5
  ITransport,
6
6
  ConnectionLocker,
7
7
  DhtAddress,
8
- getNodeIdFromPeerDescriptor
8
+ getNodeIdFromPeerDescriptor,
9
+ RingContacts
9
10
  } from '@streamr/dht'
10
11
  import {
11
12
  StreamMessage,
@@ -50,6 +51,8 @@ export interface StrictRandomGraphNodeConfig {
50
51
  nodeViewSize: number
51
52
  nearbyNodeView: NodeList
52
53
  randomNodeView: NodeList
54
+ leftNodeView: NodeList
55
+ rightNodeView: NodeList
53
56
  neighbors: NodeList
54
57
  handshaker: Handshaker
55
58
  neighborFinder: NeighborFinder
@@ -92,13 +95,15 @@ export class RandomGraphNode extends EventEmitter<Events> {
92
95
  const contact = this.config.nearbyNodeView.get(sourceId)
93
96
  || this.config.randomNodeView.get(sourceId)
94
97
  || this.config.neighbors.get(sourceId)
95
- || this.config.proxyConnectionRpcLocal?.getConnection(sourceId )?.remote
98
+ || this.config.proxyConnectionRpcLocal?.getConnection(sourceId)?.remote
96
99
  // TODO: check integrity of notifier?
97
100
  if (contact) {
98
101
  this.config.layer1Node.removeContact(sourceId)
99
102
  this.config.neighbors.remove(sourceId)
100
103
  this.config.nearbyNodeView.remove(sourceId)
101
- this.config.connectionLocker.unlockConnection(contact.getPeerDescriptor(), this.config.streamPartId)
104
+ this.config.randomNodeView.remove(sourceId)
105
+ this.config.leftNodeView.remove(sourceId)
106
+ this.config.rightNodeView.remove(sourceId)
102
107
  this.config.neighborFinder.start([sourceId])
103
108
  this.config.proxyConnectionRpcLocal?.removeConnection(sourceId)
104
109
  }
@@ -130,13 +135,29 @@ export class RandomGraphNode extends EventEmitter<Events> {
130
135
  'randomContactAdded',
131
136
  (_peerDescriptor: PeerDescriptor, randomPeers: PeerDescriptor[]) => this.onRandomContactAdded(randomPeers),
132
137
  this.abortController.signal
133
- )
138
+ )
134
139
  addManagedEventListener<any, any>(
135
140
  this.config.layer1Node as any,
136
141
  'randomContactRemoved',
137
142
  (_peerDescriptor: PeerDescriptor, randomPeers: PeerDescriptor[]) => this.onRandomContactRemoved(randomPeers),
138
143
  this.abortController.signal
139
- )
144
+ )
145
+ addManagedEventListener<any, any>(
146
+ this.config.layer1Node as any,
147
+ 'ringContactAdded',
148
+ (_: PeerDescriptor, peers: RingContacts) => {
149
+ this.onRingContactEvent(peers)
150
+ },
151
+ this.abortController.signal
152
+ )
153
+ addManagedEventListener<any, any>(
154
+ this.config.layer1Node as any,
155
+ 'ringContactRemoved',
156
+ (_: PeerDescriptor, peers: RingContacts) => {
157
+ this.onRingContactEvent(peers)
158
+ },
159
+ this.abortController.signal
160
+ )
140
161
  addManagedEventListener<any, any>(
141
162
  this.config.transport as any,
142
163
  'disconnected',
@@ -146,12 +167,27 @@ export class RandomGraphNode extends EventEmitter<Events> {
146
167
  addManagedEventListener(
147
168
  this.config.neighbors,
148
169
  'nodeAdded',
149
- (id, _remote) => {
170
+ (id, remote) => {
150
171
  this.config.propagation.onNeighborJoined(id)
172
+ this.config.connectionLocker.weakLockConnection(
173
+ getNodeIdFromPeerDescriptor(remote.getPeerDescriptor()),
174
+ this.config.streamPartId
175
+ )
151
176
  this.emit('neighborConnected', id)
152
177
  },
153
178
  this.abortController.signal
154
179
  )
180
+ addManagedEventListener(
181
+ this.config.neighbors,
182
+ 'nodeRemoved',
183
+ (_id, remote) => {
184
+ this.config.connectionLocker.weakUnlockConnection(
185
+ getNodeIdFromPeerDescriptor(remote.getPeerDescriptor()),
186
+ this.config.streamPartId
187
+ )
188
+ },
189
+ this.abortController.signal
190
+ )
155
191
  if (this.config.proxyConnectionRpcLocal !== undefined) {
156
192
  addManagedEventListener(
157
193
  this.config.proxyConnectionRpcLocal,
@@ -179,6 +215,31 @@ export class RandomGraphNode extends EventEmitter<Events> {
179
215
  (req: TemporaryConnectionRequest, context) => this.config.temporaryConnectionRpcLocal.closeConnection(req, context))
180
216
  }
181
217
 
218
+ private onRingContactEvent(ringPeers: RingContacts): void {
219
+ logger.trace(`onRingContactAdded`)
220
+ if (this.isStopped()) {
221
+ return
222
+ }
223
+ this.config.leftNodeView.replaceAll(ringPeers.left.map((peer) =>
224
+ new DeliveryRpcRemote(
225
+ this.config.localPeerDescriptor,
226
+ peer,
227
+ this.config.rpcCommunicator,
228
+ DeliveryRpcClient,
229
+ this.config.rpcRequestTimeout
230
+ )
231
+ ))
232
+ this.config.rightNodeView.replaceAll(ringPeers.right.map((peer) =>
233
+ new DeliveryRpcRemote(
234
+ this.config.localPeerDescriptor,
235
+ peer,
236
+ this.config.rpcCommunicator,
237
+ DeliveryRpcClient,
238
+ this.config.rpcRequestTimeout
239
+ )
240
+ ))
241
+ }
242
+
182
243
  private onContactAdded(closestNodes: PeerDescriptor[]): void {
183
244
  logger.trace(`New nearby contact found`)
184
245
  if (this.isStopped()) {
@@ -263,7 +324,6 @@ export class RandomGraphNode extends EventEmitter<Events> {
263
324
  const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
264
325
  if (this.config.neighbors.has(nodeId)) {
265
326
  this.config.neighbors.remove(nodeId)
266
- this.config.connectionLocker.unlockConnection(peerDescriptor, this.config.streamPartId)
267
327
  this.config.neighborFinder.start([nodeId])
268
328
  this.config.temporaryConnectionRpcLocal.removeNode(nodeId)
269
329
  }
@@ -293,9 +353,13 @@ export class RandomGraphNode extends EventEmitter<Events> {
293
353
  }
294
354
  this.abortController.abort()
295
355
  this.config.proxyConnectionRpcLocal?.stop()
296
- this.config.neighbors.getAll().map(
297
- (remote) => remote.leaveStreamPartNotice(this.config.streamPartId, this.config.isLocalNodeEntryPoint())
298
- )
356
+ this.config.neighbors.getAll().map((remote) => {
357
+ remote.leaveStreamPartNotice(this.config.streamPartId, this.config.isLocalNodeEntryPoint())
358
+ this.config.connectionLocker.weakUnlockConnection(
359
+ getNodeIdFromPeerDescriptor(remote.getPeerDescriptor()),
360
+ this.config.streamPartId
361
+ )
362
+ })
299
363
  this.config.rpcCommunicator.destroy()
300
364
  this.removeAllListeners()
301
365
  this.config.nearbyNodeView.stop()
@@ -113,8 +113,10 @@ export class StreamrNode extends EventEmitter<Events> {
113
113
  logger.debug(`Broadcasting to stream part ${streamPartId}`)
114
114
  this.joinStreamPart(streamPartId)
115
115
  this.streamParts.get(streamPartId)!.broadcast(msg)
116
- this.metrics.broadcastMessagesPerSecond.record(1)
117
- this.metrics.broadcastBytesPerSecond.record(msg.content.length)
116
+ if (msg.body.oneofKind === 'contentMessage') {
117
+ this.metrics.broadcastMessagesPerSecond.record(1)
118
+ this.metrics.broadcastBytesPerSecond.record(msg.body.contentMessage.content.length)
119
+ }
118
120
  }
119
121
 
120
122
  async leaveStreamPart(streamPartId: StreamPartID): Promise<void> {
@@ -136,7 +138,7 @@ export class StreamrNode extends EventEmitter<Events> {
136
138
  streamPartId,
137
139
  localPeerDescriptor: this.getPeerDescriptor(),
138
140
  layer1Node,
139
- getEntryPointData: (key) => this.layer0Node!.getDataFromDht(key),
141
+ fetchEntryPointData: (key) => this.layer0Node!.fetchDataFromDht(key),
140
142
  storeEntryPointData: (key, data) => this.layer0Node!.storeDataToDht(key, data),
141
143
  deleteEntryPointData: async (key) => this.layer0Node!.deleteDataFromDht(key, false)
142
144
  })
@@ -192,7 +194,10 @@ export class StreamrNode extends EventEmitter<Events> {
192
194
  entryPoints.length
193
195
  )
194
196
  entryPoints = entryPoints.concat(discoveryResult.discoveredEntryPoints)
195
- await streamPart.layer1Node.joinDht(sampleSize(entryPoints, NETWORK_SPLIT_AVOIDANCE_LIMIT))
197
+ await Promise.all([
198
+ streamPart.layer1Node.joinDht(sampleSize(entryPoints, NETWORK_SPLIT_AVOIDANCE_LIMIT)),
199
+ streamPart.layer1Node.joinRing()
200
+ ])
196
201
  if (discoveryResult.entryPointsFromDht) {
197
202
  await entryPointDiscovery.storeSelfAsEntryPointIfNecessary(entryPoints.length)
198
203
  }
@@ -13,7 +13,7 @@ import { TemporaryConnectionRpcLocal } from './temporary-connection/TemporaryCon
13
13
  import { formStreamPartDeliveryServiceId } from './formStreamPartDeliveryServiceId'
14
14
 
15
15
  type RandomGraphNodeConfig = MarkOptional<StrictRandomGraphNodeConfig,
16
- 'nearbyNodeView' | 'randomNodeView' | 'neighbors' | 'propagation'
16
+ 'nearbyNodeView' | 'randomNodeView' | 'neighbors' | 'leftNodeView' | 'rightNodeView' | 'propagation'
17
17
  | 'handshaker' | 'neighborFinder' | 'neighborUpdateManager' | 'neighborTargetCount'
18
18
  | 'rpcCommunicator' | 'nodeViewSize'
19
19
  | 'inspector' | 'temporaryConnectionRpcLocal'> & {
@@ -34,14 +34,18 @@ const createConfigWithDefaults = (config: RandomGraphNodeConfig): StrictRandomGr
34
34
  const minPropagationTargets = config.minPropagationTargets ?? 2
35
35
  const acceptProxyConnections = config.acceptProxyConnections ?? false
36
36
  const neighborUpdateInterval = config.neighborUpdateInterval ?? 10000
37
+ const neighbors = config.neighbors ?? new NodeList(ownNodeId, maxContactCount)
38
+ const leftNodeView = config.leftNodeView ?? new NodeList(ownNodeId, maxContactCount)
39
+ const rightNodeView = config.rightNodeView ?? new NodeList(ownNodeId, maxContactCount)
37
40
  const nearbyNodeView = config.nearbyNodeView ?? new NodeList(ownNodeId, maxContactCount)
38
41
  const randomNodeView = config.randomNodeView ?? new NodeList(ownNodeId, maxContactCount)
39
- const neighbors = config.neighbors ?? new NodeList(ownNodeId, maxContactCount)
40
42
  const ongoingHandshakes = new Set<DhtAddress>()
41
43
 
42
44
  const temporaryConnectionRpcLocal = new TemporaryConnectionRpcLocal({
43
45
  rpcCommunicator,
44
- localPeerDescriptor: config.localPeerDescriptor
46
+ localPeerDescriptor: config.localPeerDescriptor,
47
+ streamPartId: config.streamPartId,
48
+ connectionLocker: config.connectionLocker
45
49
  })
46
50
  const proxyConnectionRpcLocal = acceptProxyConnections ? new ProxyConnectionRpcLocal({
47
51
  localPeerDescriptor: config.localPeerDescriptor,
@@ -65,18 +69,22 @@ const createConfigWithDefaults = (config: RandomGraphNodeConfig): StrictRandomGr
65
69
  const handshaker = config.handshaker ?? new Handshaker({
66
70
  localPeerDescriptor: config.localPeerDescriptor,
67
71
  streamPartId: config.streamPartId,
68
- connectionLocker: config.connectionLocker,
69
72
  rpcCommunicator,
73
+ neighbors,
74
+ leftNodeView,
75
+ rightNodeView,
70
76
  nearbyNodeView,
71
77
  randomNodeView,
72
- neighbors,
73
78
  maxNeighborCount: neighborTargetCount,
74
79
  rpcRequestTimeout: config.rpcRequestTimeout,
75
80
  ongoingHandshakes
76
81
  })
77
82
  const neighborFinder = config.neighborFinder ?? new NeighborFinder({
78
83
  neighbors,
84
+ leftNodeView,
85
+ rightNodeView,
79
86
  nearbyNodeView,
87
+ randomNodeView,
80
88
  doFindNeighbors: (excludedIds) => handshaker.attemptHandshakesOnContacts(excludedIds),
81
89
  minCount: neighborTargetCount
82
90
  })
@@ -89,7 +97,6 @@ const createConfigWithDefaults = (config: RandomGraphNodeConfig): StrictRandomGr
89
97
  rpcCommunicator,
90
98
  neighborUpdateInterval,
91
99
  neighborTargetCount,
92
- connectionLocker: config.connectionLocker,
93
100
  ongoingHandshakes
94
101
  })
95
102
  const inspector = config.inspector ?? new Inspector({
@@ -100,9 +107,11 @@ const createConfigWithDefaults = (config: RandomGraphNodeConfig): StrictRandomGr
100
107
  })
101
108
  return {
102
109
  ...config,
110
+ neighbors,
111
+ leftNodeView,
112
+ rightNodeView,
103
113
  nearbyNodeView,
104
114
  randomNodeView,
105
- neighbors,
106
115
  rpcCommunicator,
107
116
  handshaker,
108
117
  neighborFinder,
@@ -48,7 +48,7 @@ export class Inspector {
48
48
  TemporaryConnectionRpcClient
49
49
  )
50
50
  await rpcRemote.openConnection()
51
- this.connectionLocker.lockConnection(peerDescriptor, lockId)
51
+ this.connectionLocker.weakLockConnection(getNodeIdFromPeerDescriptor(peerDescriptor), lockId)
52
52
  }
53
53
 
54
54
  async defaultCloseInspectConnection(peerDescriptor: PeerDescriptor, lockId: LockID): Promise<void> {
@@ -59,7 +59,7 @@ export class Inspector {
59
59
  TemporaryConnectionRpcClient
60
60
  )
61
61
  await rpcRemote.closeConnection()
62
- this.connectionLocker.unlockConnection(peerDescriptor, lockId)
62
+ this.connectionLocker.weakUnlockConnection(getNodeIdFromPeerDescriptor(peerDescriptor), lockId)
63
63
  }
64
64
 
65
65
  async inspect(peerDescriptor: PeerDescriptor): Promise<boolean> {
@@ -7,7 +7,6 @@ import {
7
7
  import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
8
8
  import { NodeList } from '../NodeList'
9
9
  import {
10
- ConnectionLocker,
11
10
  DhtAddress,
12
11
  DhtAddressRaw,
13
12
  DhtCallContext,
@@ -24,7 +23,6 @@ import { StreamPartID } from '@streamr/protocol'
24
23
  interface HandshakeRpcLocalConfig {
25
24
  streamPartId: StreamPartID
26
25
  neighbors: NodeList
27
- connectionLocker: ConnectionLocker
28
26
  ongoingHandshakes: Set<DhtAddress>
29
27
  ongoingInterleaves: Set<DhtAddress>
30
28
  maxNeighborCount: number
@@ -59,7 +57,10 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
59
57
  return this.acceptHandshake(request, senderDescriptor)
60
58
  } else if (this.config.neighbors.size() + this.config.ongoingHandshakes.size < this.config.maxNeighborCount) {
61
59
  return this.acceptHandshake(request, senderDescriptor)
62
- } else if (this.config.neighbors.size(getInterleaveSourceIds()) - this.config.ongoingInterleaves.size >= 2) {
60
+ } else if (
61
+ this.config.neighbors.size(getInterleaveSourceIds()) - this.config.ongoingInterleaves.size >= 2
62
+ && this.config.neighbors.size() <= this.config.maxNeighborCount
63
+ ) {
63
64
  // Do not accept the handshakes requests if the target neighbor count can potentially drop below 2
64
65
  // due to interleaving. This ensures that a stable number of connections is kept during high churn.
65
66
  return this.acceptHandshakeWithInterleaving(request, senderDescriptor)
@@ -74,7 +75,6 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
74
75
  accepted: true
75
76
  }
76
77
  this.config.neighbors.add(this.config.createDeliveryRpcRemote(requester))
77
- this.config.connectionLocker.lockConnection(requester, this.config.streamPartId)
78
78
  return res
79
79
  }
80
80
 
@@ -109,7 +109,6 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
109
109
  // If response is not accepted, keep the last node as a neighbor
110
110
  if (response.accepted) {
111
111
  this.config.neighbors.remove(getNodeIdFromPeerDescriptor(lastPeerDescriptor!))
112
- this.config.connectionLocker.unlockConnection(lastPeerDescriptor!, this.config.streamPartId)
113
112
  }
114
113
  return
115
114
  }).catch(() => {
@@ -119,7 +118,6 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
119
118
  })
120
119
  }
121
120
  this.config.neighbors.add(this.config.createDeliveryRpcRemote(requester))
122
- this.config.connectionLocker.lockConnection(requester, this.config.streamPartId)
123
121
  return {
124
122
  requestId: request.requestId,
125
123
  accepted: true,
@@ -132,10 +130,7 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
132
130
  const senderId = getNodeIdFromPeerDescriptor(senderPeerDescriptor)
133
131
  try {
134
132
  await this.config.handshakeWithInterleaving(message.interleaveTargetDescriptor!, senderId)
135
- if (this.config.neighbors.has(senderId)) {
136
- this.config.connectionLocker.unlockConnection(senderPeerDescriptor, this.config.streamPartId)
137
- this.config.neighbors.remove(senderId)
138
- }
133
+ this.config.neighbors.remove(senderId)
139
134
  return { accepted: true }
140
135
  } catch (err) {
141
136
  logger.debug(`interleaveRequest to ${getNodeIdFromPeerDescriptor(message.interleaveTargetDescriptor!)} failed: ${err}`)
@@ -1,4 +1,4 @@
1
- import { ConnectionLocker, DhtAddress, PeerDescriptor, ListeningRpcCommunicator, getNodeIdFromPeerDescriptor } from '@streamr/dht'
1
+ import { DhtAddress, PeerDescriptor, ListeningRpcCommunicator, getNodeIdFromPeerDescriptor } from '@streamr/dht'
2
2
  import { NodeList } from '../NodeList'
3
3
  import { DeliveryRpcRemote } from '../DeliveryRpcRemote'
4
4
  import {
@@ -19,8 +19,9 @@ import { StreamPartID } from '@streamr/protocol'
19
19
  interface HandshakerConfig {
20
20
  localPeerDescriptor: PeerDescriptor
21
21
  streamPartId: StreamPartID
22
- connectionLocker: ConnectionLocker
23
22
  neighbors: NodeList
23
+ leftNodeView: NodeList
24
+ rightNodeView: NodeList
24
25
  nearbyNodeView: NodeList
25
26
  randomNodeView: NodeList
26
27
  rpcCommunicator: ListeningRpcCommunicator
@@ -43,7 +44,6 @@ export class Handshaker {
43
44
  this.rpcLocal = new HandshakeRpcLocal({
44
45
  streamPartId: this.config.streamPartId,
45
46
  neighbors: this.config.neighbors,
46
- connectionLocker: this.config.connectionLocker,
47
47
  ongoingHandshakes: this.config.ongoingHandshakes,
48
48
  ongoingInterleaves: new Set(),
49
49
  maxNeighborCount: this.config.maxNeighborCount,
@@ -77,19 +77,38 @@ export class Handshaker {
77
77
  }
78
78
 
79
79
  private selectParallelTargets(excludedIds: DhtAddress[]): HandshakeRpcRemote[] {
80
- const neighbors = this.config.nearbyNodeView.getFirstAndLast(excludedIds)
80
+ const neighbors: Map<DhtAddress, DeliveryRpcRemote> = new Map()
81
+ // First add the closest left and then right contacts from the ring if possible.
82
+ const left = this.config.leftNodeView.getFirst([...excludedIds, ...Array.from(neighbors.keys())] as DhtAddress[])
83
+ const right = this.config.rightNodeView.getFirst([...excludedIds, ...Array.from(neighbors.keys())] as DhtAddress[])
84
+ if (left) {
85
+ neighbors.set(getNodeIdFromPeerDescriptor(left.getPeerDescriptor()), left)
86
+ }
87
+ if (right) {
88
+ neighbors.set(getNodeIdFromPeerDescriptor(right.getPeerDescriptor()), right)
89
+ }
90
+ // If there is still room add the closest contact based on the kademlia metric
91
+ if (neighbors.size < PARALLEL_HANDSHAKE_COUNT) {
92
+ const first = this.config.nearbyNodeView.getFirst([...excludedIds, ...Array.from(neighbors.keys())] as DhtAddress[])
93
+ if (first) {
94
+ neighbors.set(getNodeIdFromPeerDescriptor(first.getPeerDescriptor()), first)
95
+ }
96
+ }
81
97
  const getExcludedFromRandomView = () => [
82
98
  ...excludedIds,
83
- ...neighbors.map((neighbor) => getNodeIdFromPeerDescriptor(neighbor.getPeerDescriptor()))
99
+ ...Array.from(neighbors.values()).map((neighbor) => getNodeIdFromPeerDescriptor(neighbor.getPeerDescriptor()))
84
100
  ]
101
+ // If there is still room add a random contact until PARALLEL_HANDSHAKE_COUNT is reached
85
102
  while (
86
- neighbors.length < PARALLEL_HANDSHAKE_COUNT
103
+ neighbors.size < PARALLEL_HANDSHAKE_COUNT
87
104
  && this.config.randomNodeView.size(getExcludedFromRandomView()) > 0
88
105
  ) {
89
- const random = this.config.randomNodeView.getRandom(getExcludedFromRandomView())!
90
- neighbors.push(random)
106
+ const random = this.config.randomNodeView.getRandom([...excludedIds, ...Array.from(neighbors.keys())] as DhtAddress[])
107
+ if (random) {
108
+ neighbors.set(getNodeIdFromPeerDescriptor(random.getPeerDescriptor()), random)
109
+ }
91
110
  }
92
- return neighbors.map((neighbor) => this.createRpcRemote(neighbor.getPeerDescriptor()))
111
+ return Array.from(neighbors.values()).map((neighbor) => this.createRpcRemote(neighbor.getPeerDescriptor()))
93
112
  }
94
113
 
95
114
  private async doParallelHandshakes(targets: HandshakeRpcRemote[], excludedIds: DhtAddress[]): Promise<DhtAddress[]> {
@@ -111,7 +130,10 @@ export class Handshaker {
111
130
 
112
131
  private async selectNewTargetAndHandshake(excludedIds: DhtAddress[]): Promise<DhtAddress[]> {
113
132
  const exclude = excludedIds.concat(this.config.neighbors.getIds())
114
- const neighbor = this.config.nearbyNodeView.getFirst(exclude) ?? this.config.randomNodeView.getRandom(exclude)
133
+ const neighbor = this.config.leftNodeView.getFirst(exclude)
134
+ ?? this.config.rightNodeView.getFirst(exclude)
135
+ ?? this.config.nearbyNodeView.getFirst(exclude)
136
+ ?? this.config.randomNodeView.getRandom(exclude)
115
137
  if (neighbor) {
116
138
  const accepted = await this.handshakeWithTarget(this.createRpcRemote(neighbor.getPeerDescriptor()))
117
139
  if (!accepted) {
@@ -131,7 +153,6 @@ export class Handshaker {
131
153
  )
132
154
  if (result.accepted) {
133
155
  this.config.neighbors.add(this.createDeliveryRpcRemote(neighbor.getPeerDescriptor()))
134
- this.config.connectionLocker.lockConnection(neighbor.getPeerDescriptor(), this.config.streamPartId)
135
156
  }
136
157
  if (result.interleaveTargetDescriptor) {
137
158
  await this.handshakeWithInterleaving(result.interleaveTargetDescriptor, targetNodeId)
@@ -152,7 +173,6 @@ export class Handshaker {
152
173
  )
153
174
  if (result.accepted) {
154
175
  this.config.neighbors.add(this.createDeliveryRpcRemote(neighbor.getPeerDescriptor()))
155
- this.config.connectionLocker.lockConnection(neighbor.getPeerDescriptor(), this.config.streamPartId)
156
176
  }
157
177
  this.config.ongoingHandshakes.delete(targetNodeId)
158
178
  return result.accepted
@@ -5,6 +5,9 @@ import { DhtAddress } from '@streamr/dht'
5
5
  interface FindNeighborsSessionConfig {
6
6
  neighbors: NodeList
7
7
  nearbyNodeView: NodeList
8
+ leftNodeView: NodeList
9
+ rightNodeView: NodeList
10
+ randomNodeView: NodeList
8
11
  doFindNeighbors: (excludedNodes: DhtAddress[]) => Promise<DhtAddress[]>
9
12
  minCount: number
10
13
  }
@@ -27,7 +30,13 @@ export class NeighborFinder {
27
30
  return
28
31
  }
29
32
  const newExcludes = await this.config.doFindNeighbors(excluded)
30
- if (this.config.neighbors.size() < this.config.minCount && newExcludes.length < this.config.nearbyNodeView.size()) {
33
+ const uniqueContactCount = new Set([
34
+ ...this.config.nearbyNodeView.getIds(),
35
+ ...this.config.leftNodeView.getIds(),
36
+ ...this.config.rightNodeView.getIds(),
37
+ ...this.config.randomNodeView.getIds()
38
+ ]).size
39
+ if (this.config.neighbors.size() < this.config.minCount && newExcludes.length < uniqueContactCount) {
31
40
  // TODO should we catch possible promise rejection?
32
41
  setAbortableTimeout(() => this.findNeighbors(newExcludes), INTERVAL, this.abortController.signal)
33
42
  } else {