@streamr/dht 100.0.0-testnet-three.2 → 100.0.0-testnet-three.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -8
- package/dist/package.json +5 -5
- package/dist/src/connection/ConnectionManager.js +10 -2
- package/dist/src/connection/ConnectionManager.js.map +1 -1
- package/dist/src/connection/connectivityRequestHandler.d.ts +1 -0
- package/dist/src/connection/connectivityRequestHandler.js +39 -22
- package/dist/src/connection/connectivityRequestHandler.js.map +1 -1
- package/dist/src/connection/websocket/WebsocketConnector.js +32 -37
- package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
- package/dist/src/dht/DhtNode.d.ts +2 -2
- package/dist/src/dht/DhtNode.js +16 -11
- package/dist/src/dht/DhtNode.js.map +1 -1
- package/dist/src/dht/DhtNodeRpcLocal.d.ts +1 -1
- package/dist/src/dht/DhtNodeRpcLocal.js +2 -2
- package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
- package/dist/src/dht/PeerManager.d.ts +7 -9
- package/dist/src/dht/PeerManager.js +7 -14
- package/dist/src/dht/PeerManager.js.map +1 -1
- package/dist/src/dht/contact/ContactList.d.ts +1 -1
- package/dist/src/dht/contact/RandomContactList.js +1 -1
- package/dist/src/dht/contact/RandomContactList.js.map +1 -1
- package/dist/src/dht/contact/SortedContactList.js +14 -15
- package/dist/src/dht/contact/SortedContactList.js.map +1 -1
- package/dist/src/dht/discovery/DiscoverySession.d.ts +1 -1
- package/dist/src/dht/discovery/DiscoverySession.js +5 -5
- package/dist/src/dht/discovery/DiscoverySession.js.map +1 -1
- package/dist/src/dht/discovery/PeerDiscovery.js +2 -2
- package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationRpcLocal.js +0 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationRpcLocal.js.map +1 -1
- package/dist/src/dht/routing/RouterRpcLocal.js +2 -3
- package/dist/src/dht/routing/RouterRpcLocal.js.map +1 -1
- package/dist/src/dht/routing/RoutingSession.js +2 -1
- package/dist/src/dht/routing/RoutingSession.js.map +1 -1
- package/dist/src/dht/store/LocalDataStore.d.ts +1 -1
- package/dist/src/dht/store/LocalDataStore.js +2 -5
- package/dist/src/dht/store/LocalDataStore.js.map +1 -1
- package/dist/src/dht/store/StoreManager.d.ts +1 -1
- package/dist/src/dht/store/StoreManager.js +10 -17
- package/dist/src/dht/store/StoreManager.js.map +1 -1
- package/package.json +5 -5
- package/src/connection/ConnectionManager.ts +9 -2
- package/src/connection/connectivityRequestHandler.ts +39 -22
- package/src/connection/websocket/WebsocketConnector.ts +33 -37
- package/src/dht/DhtNode.ts +20 -15
- package/src/dht/DhtNodeRpcLocal.ts +3 -3
- package/src/dht/PeerManager.ts +12 -21
- package/src/dht/contact/ContactList.ts +1 -1
- package/src/dht/contact/RandomContactList.ts +1 -1
- package/src/dht/contact/SortedContactList.ts +14 -18
- package/src/dht/discovery/DiscoverySession.ts +5 -5
- package/src/dht/discovery/PeerDiscovery.ts +2 -2
- package/src/dht/recursive-operation/RecursiveOperationRpcLocal.ts +0 -1
- package/src/dht/routing/RouterRpcLocal.ts +2 -3
- package/src/dht/routing/RoutingSession.ts +2 -1
- package/src/dht/store/LocalDataStore.ts +2 -5
- package/src/dht/store/StoreManager.ts +10 -17
- package/test/end-to-end/Layer0Webrtc-Layer1.test.ts +8 -4
- package/test/end-to-end/Layer0Webrtc.test.ts +0 -2
- package/test/end-to-end/memory-leak.test.ts +3 -3
- package/test/unit/PeerManager.test.ts +1 -1
- package/test/unit/SortedContactList.test.ts +5 -5
- package/test/unit/StoreManager.test.ts +26 -23
- package/test/unit/connectivityRequestHandler.test.ts +39 -6
package/src/dht/PeerManager.ts
CHANGED
|
@@ -25,9 +25,9 @@ interface PeerManagerConfig {
|
|
|
25
25
|
}
|
|
26
26
|
|
|
27
27
|
export interface PeerManagerEvents {
|
|
28
|
-
|
|
28
|
+
contactAdded: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
|
|
29
29
|
contactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
|
|
30
|
-
|
|
30
|
+
randomContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
|
|
31
31
|
randomContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
|
|
32
32
|
kBucketEmpty: () => void
|
|
33
33
|
}
|
|
@@ -81,15 +81,15 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
81
81
|
this.emit('contactRemoved', removedContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
|
|
82
82
|
this.randomPeers.addContact(this.config.createDhtNodeRpcRemote(removedContact.getPeerDescriptor()))
|
|
83
83
|
})
|
|
84
|
-
this.contacts.on('
|
|
85
|
-
this.emit('
|
|
84
|
+
this.contacts.on('contactAdded', (contactAdded: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
|
|
85
|
+
this.emit('contactAdded', contactAdded.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
|
|
86
86
|
)
|
|
87
87
|
this.randomPeers = new RandomContactList(this.config.localNodeId, this.config.maxContactListSize)
|
|
88
88
|
this.randomPeers.on('contactRemoved', (removedContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
|
|
89
89
|
this.emit('randomContactRemoved', removedContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
|
|
90
90
|
)
|
|
91
|
-
this.randomPeers.on('
|
|
92
|
-
this.emit('
|
|
91
|
+
this.randomPeers.on('contactAdded', (contactAdded: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
|
|
92
|
+
this.emit('randomContactAdded', contactAdded.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
|
|
93
93
|
)
|
|
94
94
|
}
|
|
95
95
|
|
|
@@ -160,7 +160,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
160
160
|
}
|
|
161
161
|
const closest = this.getClosestActiveContactNotInBucket()
|
|
162
162
|
if (closest) {
|
|
163
|
-
this.
|
|
163
|
+
this.addContact([closest.getPeerDescriptor()])
|
|
164
164
|
}
|
|
165
165
|
}
|
|
166
166
|
|
|
@@ -173,7 +173,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
173
173
|
return undefined
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
|
|
176
|
+
onContactConnected(peerDescriptor: PeerDescriptor): void {
|
|
177
177
|
const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
|
|
178
178
|
if (nodeId === this.config.localNodeId) {
|
|
179
179
|
logger.error('handleConnected() to self')
|
|
@@ -188,7 +188,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
188
188
|
logger.trace('connected: ' + nodeId + ' ' + this.connections.size)
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
-
|
|
191
|
+
onContactDisconnected(nodeId: DhtAddress, gracefulLeave: boolean): void {
|
|
192
192
|
logger.trace('disconnected: ' + nodeId)
|
|
193
193
|
this.connections.delete(nodeId)
|
|
194
194
|
if (this.config.isLayer0) {
|
|
@@ -202,11 +202,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
202
202
|
}
|
|
203
203
|
}
|
|
204
204
|
|
|
205
|
-
|
|
206
|
-
this.removeContact(nodeId)
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
private removeContact(nodeId: DhtAddress): void {
|
|
205
|
+
removeContact(nodeId: DhtAddress): void {
|
|
210
206
|
if (this.stopped) {
|
|
211
207
|
return
|
|
212
208
|
}
|
|
@@ -274,16 +270,11 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
274
270
|
return this.bucket.toArray().map((rpcRemote: DhtNodeRpcRemote) => rpcRemote.getPeerDescriptor())
|
|
275
271
|
}
|
|
276
272
|
|
|
277
|
-
|
|
273
|
+
setContactActive(nodeId: DhtAddress): void {
|
|
278
274
|
this.contacts.setActive(nodeId)
|
|
279
275
|
}
|
|
280
276
|
|
|
281
|
-
|
|
282
|
-
this.bucket.remove(getRawFromDhtAddress(nodeId))
|
|
283
|
-
this.contacts.removeContact(nodeId)
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
handleNewPeers(peerDescriptors: PeerDescriptor[], setActive?: boolean): void {
|
|
277
|
+
addContact(peerDescriptors: PeerDescriptor[], setActive?: boolean): void {
|
|
287
278
|
if (this.stopped) {
|
|
288
279
|
return
|
|
289
280
|
}
|
|
@@ -13,7 +13,7 @@ export class ContactState<C> {
|
|
|
13
13
|
|
|
14
14
|
export interface Events<C> {
|
|
15
15
|
contactRemoved: (removedContact: C, closestContacts: C[]) => void
|
|
16
|
-
|
|
16
|
+
contactAdded: (contactAdded: C, closestContacts: C[]) => void
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
export class ContactList<C extends { getNodeId: () => DhtAddress }> extends EventEmitter<Events<C>> {
|
|
@@ -29,7 +29,7 @@ export class RandomContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
29
29
|
this.contactIds.push(contact.getNodeId())
|
|
30
30
|
this.contactsById.set(contact.getNodeId(), new ContactState(contact))
|
|
31
31
|
this.emit(
|
|
32
|
-
'
|
|
32
|
+
'contactAdded',
|
|
33
33
|
contact,
|
|
34
34
|
this.getContacts()
|
|
35
35
|
)
|
|
@@ -40,37 +40,33 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
public addContact(contact: C): void {
|
|
43
|
-
|
|
43
|
+
const contactId = contact.getNodeId()
|
|
44
|
+
if (this.config.excludedNodeIds !== undefined && this.config.excludedNodeIds.has(contactId)) {
|
|
44
45
|
return
|
|
45
46
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
(this.config.nodeIdDistanceLimit !== undefined && this.compareIds(this.config.nodeIdDistanceLimit, contact.getNodeId()) < 0)) {
|
|
47
|
+
if ((!this.config.allowToContainReferenceId && (this.config.referenceId === contactId)) ||
|
|
48
|
+
(this.config.nodeIdDistanceLimit !== undefined && this.compareIds(this.config.nodeIdDistanceLimit, contactId) < 0)) {
|
|
49
49
|
return
|
|
50
50
|
}
|
|
51
|
-
if (!this.contactsById.has(
|
|
51
|
+
if (!this.contactsById.has(contactId)) {
|
|
52
52
|
if ((this.config.maxSize === undefined) || (this.contactIds.length < this.config.maxSize)) {
|
|
53
|
-
this.contactsById.set(
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
const index = sortedIndexBy(this.contactIds, contact.getNodeId(), (id: DhtAddress) => { return this.distanceToReferenceId(id) })
|
|
57
|
-
this.contactIds.splice(index, 0, contact.getNodeId())
|
|
53
|
+
this.contactsById.set(contactId, new ContactState(contact))
|
|
54
|
+
const index = sortedIndexBy(this.contactIds, contactId, (id: DhtAddress) => { return this.distanceToReferenceId(id) })
|
|
55
|
+
this.contactIds.splice(index, 0, contactId)
|
|
58
56
|
if (this.config.emitEvents) {
|
|
59
57
|
this.emit(
|
|
60
|
-
'
|
|
58
|
+
'contactAdded',
|
|
61
59
|
contact,
|
|
62
60
|
this.getClosestContacts()
|
|
63
61
|
)
|
|
64
62
|
}
|
|
65
|
-
} else if (this.compareIds(this.contactIds[this.config.maxSize - 1],
|
|
63
|
+
} else if (this.compareIds(this.contactIds[this.config.maxSize - 1], contactId) > 0) {
|
|
66
64
|
const removedId = this.contactIds.pop()
|
|
67
65
|
const removedContact = this.contactsById.get(removedId!)!.contact
|
|
68
66
|
this.contactsById.delete(removedId!)
|
|
69
|
-
this.contactsById.set(
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const index = sortedIndexBy(this.contactIds, contact.getNodeId(), (id: DhtAddress) => { return this.distanceToReferenceId(id) })
|
|
73
|
-
this.contactIds.splice(index, 0, contact.getNodeId())
|
|
67
|
+
this.contactsById.set(contactId, new ContactState(contact))
|
|
68
|
+
const index = sortedIndexBy(this.contactIds, contactId, (id: DhtAddress) => { return this.distanceToReferenceId(id) })
|
|
69
|
+
this.contactIds.splice(index, 0, contactId)
|
|
74
70
|
if (this.config.emitEvents) {
|
|
75
71
|
const closestContacts = this.getClosestContacts()
|
|
76
72
|
this.emit(
|
|
@@ -79,7 +75,7 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
79
75
|
closestContacts
|
|
80
76
|
)
|
|
81
77
|
this.emit(
|
|
82
|
-
'
|
|
78
|
+
'contactAdded',
|
|
83
79
|
contact,
|
|
84
80
|
closestContacts
|
|
85
81
|
)
|
|
@@ -34,11 +34,11 @@ export class DiscoverySession {
|
|
|
34
34
|
this.config = config
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
private
|
|
37
|
+
private addContacts(contacts: PeerDescriptor[]): void {
|
|
38
38
|
if (this.stopped) {
|
|
39
39
|
return
|
|
40
40
|
}
|
|
41
|
-
this.config.peerManager.
|
|
41
|
+
this.config.peerManager.addContact(contacts)
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
private async getClosestPeersFromContact(contact: DhtNodeRpcRemote): Promise<PeerDescriptor[]> {
|
|
@@ -48,7 +48,7 @@ export class DiscoverySession {
|
|
|
48
48
|
logger.trace(`Getting closest peers from contact: ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
|
|
49
49
|
this.config.contactedPeers.add(contact.getNodeId())
|
|
50
50
|
const returnedContacts = await contact.getClosestPeers(this.config.targetId)
|
|
51
|
-
this.config.peerManager.
|
|
51
|
+
this.config.peerManager.setContactActive(contact.getNodeId())
|
|
52
52
|
return returnedContacts
|
|
53
53
|
}
|
|
54
54
|
|
|
@@ -60,7 +60,7 @@ export class DiscoverySession {
|
|
|
60
60
|
const targetId = getRawFromDhtAddress(this.config.targetId)
|
|
61
61
|
const oldClosestNeighbor = this.config.peerManager.getClosestNeighborsTo(this.config.targetId, 1)[0]
|
|
62
62
|
const oldClosestDistance = getDistance(targetId, getRawFromDhtAddress(oldClosestNeighbor.getNodeId()))
|
|
63
|
-
this.
|
|
63
|
+
this.addContacts(contacts)
|
|
64
64
|
const newClosestNeighbor = this.config.peerManager.getClosestNeighborsTo(this.config.targetId, 1)[0]
|
|
65
65
|
const newClosestDistance = getDistance(targetId, getRawFromDhtAddress(newClosestNeighbor.getNodeId()))
|
|
66
66
|
if (newClosestDistance >= oldClosestDistance) {
|
|
@@ -73,7 +73,7 @@ export class DiscoverySession {
|
|
|
73
73
|
return
|
|
74
74
|
}
|
|
75
75
|
this.ongoingClosestPeersRequests.delete(peer.getNodeId())
|
|
76
|
-
this.config.peerManager.
|
|
76
|
+
this.config.peerManager.removeContact(peer.getNodeId())
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
private findMoreContacts(): void {
|
|
@@ -75,7 +75,7 @@ export class PeerDiscovery {
|
|
|
75
75
|
return
|
|
76
76
|
}
|
|
77
77
|
this.config.connectionManager?.lockConnection(entryPointDescriptor, `${this.config.serviceId}::joinDht`)
|
|
78
|
-
this.config.peerManager.
|
|
78
|
+
this.config.peerManager.addContact([entryPointDescriptor])
|
|
79
79
|
const targetId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
|
|
80
80
|
const sessions = [this.createSession(targetId, contactedPeers)]
|
|
81
81
|
if (additionalDistantJoin.enabled) {
|
|
@@ -162,7 +162,7 @@ export class PeerDiscovery {
|
|
|
162
162
|
await Promise.allSettled(
|
|
163
163
|
nodes.map(async (peer: DhtNodeRpcRemote) => {
|
|
164
164
|
const contacts = await peer.getClosestPeers(localNodeId)
|
|
165
|
-
this.config.peerManager.
|
|
165
|
+
this.config.peerManager.addContact(contacts)
|
|
166
166
|
})
|
|
167
167
|
)
|
|
168
168
|
}
|
|
@@ -28,7 +28,6 @@ export class RecursiveOperationRpcLocal implements IRecursiveOperationRpc {
|
|
|
28
28
|
}
|
|
29
29
|
const senderId = getNodeIdFromPeerDescriptor(getPreviousPeer(routedMessage) ?? routedMessage.sourcePeer!)
|
|
30
30
|
logger.trace(`Received routeRequest call from ${senderId}`)
|
|
31
|
-
this.config.addContact(routedMessage.sourcePeer!, true)
|
|
32
31
|
this.config.addToDuplicateDetector(routedMessage.requestId)
|
|
33
32
|
return this.config.doRouteRequest(routedMessage)
|
|
34
33
|
}
|
|
@@ -5,6 +5,7 @@ import { IRouterRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
|
|
|
5
5
|
import { DuplicateDetector } from './DuplicateDetector'
|
|
6
6
|
import { RoutingMode } from './RoutingSession'
|
|
7
7
|
import { areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor } from '../../identifiers'
|
|
8
|
+
import { v4 } from 'uuid'
|
|
8
9
|
|
|
9
10
|
interface RouterRpcLocalConfig {
|
|
10
11
|
doRouteMessage: (routedMessage: RouteMessageWrapper, mode?: RoutingMode) => RouteMessageAck
|
|
@@ -40,7 +41,6 @@ export class RouterRpcLocal implements IRouterRpc {
|
|
|
40
41
|
return createRouteMessageAck(routedMessage, RouteMessageError.DUPLICATE)
|
|
41
42
|
}
|
|
42
43
|
logger.trace(`Processing received routeMessage ${routedMessage.requestId}`)
|
|
43
|
-
this.config.addContact(routedMessage.sourcePeer!, true)
|
|
44
44
|
this.config.duplicateRequestDetector.add(routedMessage.requestId)
|
|
45
45
|
if (areEqualBinaries(this.config.localPeerDescriptor.nodeId, routedMessage.target)) {
|
|
46
46
|
logger.trace(`routing message targeted to self ${routedMessage.requestId}`)
|
|
@@ -59,7 +59,6 @@ export class RouterRpcLocal implements IRouterRpc {
|
|
|
59
59
|
return createRouteMessageAck(forwardMessage, RouteMessageError.DUPLICATE)
|
|
60
60
|
}
|
|
61
61
|
logger.trace(`Processing received forward routeMessage ${forwardMessage.requestId}`)
|
|
62
|
-
this.config.addContact(forwardMessage.sourcePeer!, true)
|
|
63
62
|
this.config.duplicateRequestDetector.add(forwardMessage.requestId)
|
|
64
63
|
if (areEqualBinaries(this.config.localPeerDescriptor.nodeId, forwardMessage.target)) {
|
|
65
64
|
return this.forwardToDestination(forwardMessage)
|
|
@@ -75,7 +74,7 @@ export class RouterRpcLocal implements IRouterRpc {
|
|
|
75
74
|
this.config.connectionManager?.handleMessage(forwardedMessage)
|
|
76
75
|
return createRouteMessageAck(routedMessage)
|
|
77
76
|
}
|
|
78
|
-
return this.config.doRouteMessage({ ...routedMessage, target: forwardedMessage.targetDescriptor!.nodeId })
|
|
77
|
+
return this.config.doRouteMessage({ ...routedMessage, requestId: v4(), target: forwardedMessage.targetDescriptor!.nodeId })
|
|
79
78
|
}
|
|
80
79
|
|
|
81
80
|
}
|
|
@@ -17,6 +17,7 @@ import { pull } from 'lodash'
|
|
|
17
17
|
const logger = new Logger(module)
|
|
18
18
|
|
|
19
19
|
const MAX_FAILED_HOPS = 2
|
|
20
|
+
const CONTACT_LIST_MAX_SIZE = 10
|
|
20
21
|
|
|
21
22
|
class RemoteContact extends Contact {
|
|
22
23
|
|
|
@@ -90,7 +91,7 @@ export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
|
|
|
90
91
|
const previousId = previousPeer ? getNodeIdFromPeerDescriptor(previousPeer) : undefined
|
|
91
92
|
this.contactList = new SortedContactList({
|
|
92
93
|
referenceId: getDhtAddressFromRaw(config.routedMessage.target),
|
|
93
|
-
maxSize:
|
|
94
|
+
maxSize: CONTACT_LIST_MAX_SIZE,
|
|
94
95
|
allowToContainReferenceId: true,
|
|
95
96
|
nodeIdDistanceLimit: previousId,
|
|
96
97
|
excludedNodeIds: config.excludedNodeIds,
|
|
@@ -57,11 +57,8 @@ export class LocalDataStore {
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
public
|
|
61
|
-
|
|
62
|
-
if (storedEntry) {
|
|
63
|
-
storedEntry.stale = stale
|
|
64
|
-
}
|
|
60
|
+
public keys(): IterableIterator<DhtAddress> {
|
|
61
|
+
return this.store.keys()
|
|
65
62
|
}
|
|
66
63
|
|
|
67
64
|
public setAllEntriesAsStale(key: DhtAddress): void {
|
|
@@ -13,7 +13,6 @@ import { Timestamp } from '../../proto/google/protobuf/timestamp'
|
|
|
13
13
|
import { SortedContactList } from '../contact/SortedContactList'
|
|
14
14
|
import { Contact } from '../contact/Contact'
|
|
15
15
|
import { ServiceID } from '../../types/ServiceID'
|
|
16
|
-
import { findIndex } from 'lodash'
|
|
17
16
|
import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
|
|
18
17
|
import { StoreRpcLocal } from './StoreRpcLocal'
|
|
19
18
|
import { getDistance } from '../PeerManager'
|
|
@@ -53,20 +52,18 @@ export class StoreManager {
|
|
|
53
52
|
(request: ReplicateDataRequest, context: ServerCallContext) => rpcLocal.replicateData(request, context))
|
|
54
53
|
}
|
|
55
54
|
|
|
56
|
-
|
|
57
|
-
for (const
|
|
58
|
-
this.replicateAndUpdateStaleState(
|
|
55
|
+
onContactAdded(peerDescriptor: PeerDescriptor): void {
|
|
56
|
+
for (const key of this.config.localDataStore.keys()) {
|
|
57
|
+
this.replicateAndUpdateStaleState(key, peerDescriptor)
|
|
59
58
|
}
|
|
60
59
|
}
|
|
61
60
|
|
|
62
|
-
private replicateAndUpdateStaleState(
|
|
61
|
+
private replicateAndUpdateStaleState(key: DhtAddress, newNode: PeerDescriptor): void {
|
|
63
62
|
const newNodeId = getNodeIdFromPeerDescriptor(newNode)
|
|
64
|
-
const
|
|
65
|
-
// TODO use config option or named constant?
|
|
66
|
-
const closestToData = this.config.getClosestNeighborsTo(key, 10)
|
|
63
|
+
const closestToData = this.config.getClosestNeighborsTo(key, this.config.redundancyFactor)
|
|
67
64
|
const sortedList = new SortedContactList<Contact>({
|
|
68
65
|
referenceId: key,
|
|
69
|
-
maxSize:
|
|
66
|
+
maxSize: this.config.redundancyFactor,
|
|
70
67
|
allowToContainReferenceId: true,
|
|
71
68
|
emitEvents: false
|
|
72
69
|
})
|
|
@@ -79,18 +76,14 @@ export class StoreManager {
|
|
|
79
76
|
const selfIsPrimaryStorer = (sortedList.getClosestContactId() === getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor))
|
|
80
77
|
if (selfIsPrimaryStorer) {
|
|
81
78
|
sortedList.addContact(new Contact(newNode))
|
|
82
|
-
|
|
83
|
-
// findIndex should never return -1 here because we just added the new node to the list
|
|
84
|
-
const index = findIndex(sorted, (nodeId) => (nodeId === newNodeId))
|
|
85
|
-
// if new node is within the storageRedundancyFactor closest nodes to the data
|
|
86
|
-
// do replicate data to it
|
|
87
|
-
if (index < this.config.redundancyFactor) {
|
|
79
|
+
if (sortedList.getContact(newNodeId) !== undefined) {
|
|
88
80
|
setImmediate(async () => {
|
|
89
|
-
|
|
81
|
+
const dataEntries = Array.from(this.config.localDataStore.values(key))
|
|
82
|
+
await Promise.all(dataEntries.map(async (dataEntry) => this.replicateDataToContact(dataEntry, newNode)))
|
|
90
83
|
})
|
|
91
84
|
}
|
|
92
85
|
} else if (!this.selfIsWithinRedundancyFactor(key)) {
|
|
93
|
-
this.config.localDataStore.
|
|
86
|
+
this.config.localDataStore.setAllEntriesAsStale(key)
|
|
94
87
|
}
|
|
95
88
|
}
|
|
96
89
|
|
|
@@ -28,22 +28,26 @@ describe('Layer 1 on Layer 0 with mocked connections', () => {
|
|
|
28
28
|
|
|
29
29
|
const layer0Node1Id = createRandomDhtAddress()
|
|
30
30
|
layer0Node1 = new DhtNode({
|
|
31
|
-
nodeId: layer0Node1Id
|
|
31
|
+
nodeId: layer0Node1Id,
|
|
32
|
+
entryPoints: [entrypointDescriptor]
|
|
32
33
|
})
|
|
33
34
|
|
|
34
35
|
const layer0Node2Id = createRandomDhtAddress()
|
|
35
36
|
layer0Node2 = new DhtNode({
|
|
36
|
-
nodeId: layer0Node2Id
|
|
37
|
+
nodeId: layer0Node2Id,
|
|
38
|
+
entryPoints: [entrypointDescriptor]
|
|
37
39
|
})
|
|
38
40
|
|
|
39
41
|
const layer0Node3Id = createRandomDhtAddress()
|
|
40
42
|
layer0Node3 = new DhtNode({
|
|
41
|
-
nodeId: layer0Node3Id
|
|
43
|
+
nodeId: layer0Node3Id,
|
|
44
|
+
entryPoints: [entrypointDescriptor]
|
|
42
45
|
})
|
|
43
46
|
|
|
44
47
|
const layer0Node4Id = createRandomDhtAddress()
|
|
45
48
|
layer0Node4 = new DhtNode({
|
|
46
|
-
nodeId: layer0Node4Id
|
|
49
|
+
nodeId: layer0Node4Id,
|
|
50
|
+
entryPoints: [entrypointDescriptor]
|
|
47
51
|
})
|
|
48
52
|
|
|
49
53
|
layer1EntryPoint = new DhtNode({
|
|
@@ -30,8 +30,8 @@ describe('memory leak', () => {
|
|
|
30
30
|
})
|
|
31
31
|
await entryPoint.start()
|
|
32
32
|
await entryPoint.joinDht([entryPointDescriptor])
|
|
33
|
-
let sender: DhtNode | undefined = new DhtNode({})
|
|
34
|
-
let receiver: DhtNode | undefined = new DhtNode({})
|
|
33
|
+
let sender: DhtNode | undefined = new DhtNode({ entryPoints: [entryPointDescriptor] })
|
|
34
|
+
let receiver: DhtNode | undefined = new DhtNode({ entryPoints: [entryPointDescriptor] })
|
|
35
35
|
await Promise.all([
|
|
36
36
|
(async () => {
|
|
37
37
|
await sender.start()
|
|
@@ -77,5 +77,5 @@ describe('memory leak', () => {
|
|
|
77
77
|
const detector3 = new LeakDetector(receiver)
|
|
78
78
|
receiver = undefined
|
|
79
79
|
expect(await detector3.isLeaking()).toBe(false)
|
|
80
|
-
})
|
|
80
|
+
}, 10000)
|
|
81
81
|
})
|
|
@@ -15,7 +15,7 @@ describe('PeerManager', () => {
|
|
|
15
15
|
return new DhtNodeRpcRemote(undefined as any, peerDescriptor, undefined as any, new MockRpcCommunicator())
|
|
16
16
|
}
|
|
17
17
|
} as any)
|
|
18
|
-
manager.
|
|
18
|
+
manager.addContact(nodeIds.map((n) => ({ nodeId: getRawFromDhtAddress(n), type: NodeType.NODEJS })))
|
|
19
19
|
|
|
20
20
|
const referenceId = createRandomDhtAddress()
|
|
21
21
|
const excluded = new Set<DhtAddress>(sampleSize(nodeIds, 2)!)
|
|
@@ -119,15 +119,15 @@ describe('SortedContactList', () => {
|
|
|
119
119
|
expect(list.getActiveContacts()).toEqual([item2, item3, item4])
|
|
120
120
|
})
|
|
121
121
|
|
|
122
|
-
it('does not emit
|
|
122
|
+
it('does not emit contactAdded if contact did not fit the structure', () => {
|
|
123
123
|
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 2, allowToContainReferenceId: false, emitEvents: true })
|
|
124
|
-
const
|
|
125
|
-
list.on('
|
|
124
|
+
const onContactAdded = jest.fn()
|
|
125
|
+
list.on('contactAdded', onContactAdded)
|
|
126
126
|
list.addContact(item1)
|
|
127
127
|
list.addContact(item2)
|
|
128
|
-
expect(
|
|
128
|
+
expect(onContactAdded).toBeCalledTimes(2)
|
|
129
129
|
list.addContact(item3)
|
|
130
|
-
expect(
|
|
130
|
+
expect(onContactAdded).toBeCalledTimes(2)
|
|
131
131
|
expect(list.getAllContacts().length).toEqual(2)
|
|
132
132
|
})
|
|
133
133
|
|
|
@@ -27,7 +27,7 @@ describe('StoreManager', () => {
|
|
|
27
27
|
localNodeId: DhtAddress,
|
|
28
28
|
closestNeighbors: DhtAddress[],
|
|
29
29
|
replicateData: (request: ReplicateDataRequest) => unknown,
|
|
30
|
-
|
|
30
|
+
setAllEntriesAsStale: (key: DhtAddress) => unknown
|
|
31
31
|
): StoreManager => {
|
|
32
32
|
const getClosestNeighborsTo = () => {
|
|
33
33
|
return closestNeighbors.map((nodeId) => ({ nodeId: getRawFromDhtAddress(nodeId), type: NodeType.NODEJS }))
|
|
@@ -39,7 +39,11 @@ describe('StoreManager', () => {
|
|
|
39
39
|
} as any,
|
|
40
40
|
recursiveOperationManager: undefined as any,
|
|
41
41
|
localPeerDescriptor: { nodeId: getRawFromDhtAddress(localNodeId), type: NodeType.NODEJS },
|
|
42
|
-
localDataStore: {
|
|
42
|
+
localDataStore: {
|
|
43
|
+
keys: () => [getDhtAddressFromRaw(DATA_ENTRY.key)],
|
|
44
|
+
values: () => [DATA_ENTRY],
|
|
45
|
+
setAllEntriesAsStale
|
|
46
|
+
} as any,
|
|
43
47
|
serviceId: undefined as any,
|
|
44
48
|
highestTtl: undefined as any,
|
|
45
49
|
redundancyFactor: 3,
|
|
@@ -52,34 +56,34 @@ describe('StoreManager', () => {
|
|
|
52
56
|
|
|
53
57
|
it('new node is within redundancy factor', async () => {
|
|
54
58
|
const replicateData = jest.fn<undefined, [ReplicateDataRequest]>()
|
|
55
|
-
const
|
|
59
|
+
const setAllEntriesAsStale = jest.fn<undefined, [DhtAddress]>()
|
|
56
60
|
const manager = createStoreManager(
|
|
57
61
|
NODES_CLOSEST_TO_DATA[0],
|
|
58
62
|
[NODES_CLOSEST_TO_DATA[1], NODES_CLOSEST_TO_DATA[3], NODES_CLOSEST_TO_DATA[4]],
|
|
59
63
|
replicateData,
|
|
60
|
-
|
|
64
|
+
setAllEntriesAsStale
|
|
61
65
|
)
|
|
62
|
-
manager.
|
|
66
|
+
manager.onContactAdded({ nodeId: getRawFromDhtAddress(NODES_CLOSEST_TO_DATA[2]), type: NodeType.NODEJS })
|
|
63
67
|
await waitForCondition(() => replicateData.mock.calls.length === 1)
|
|
64
68
|
expect(replicateData).toHaveBeenCalledWith({
|
|
65
69
|
entry: DATA_ENTRY
|
|
66
70
|
})
|
|
67
|
-
expect(
|
|
71
|
+
expect(setAllEntriesAsStale).not.toHaveBeenCalled()
|
|
68
72
|
})
|
|
69
73
|
|
|
70
74
|
it('new node is not within redundancy factor', async () => {
|
|
71
75
|
const replicateData = jest.fn<undefined, [ReplicateDataRequest]>()
|
|
72
|
-
const
|
|
76
|
+
const setAllEntriesAsStale = jest.fn<undefined, [DhtAddress]>()
|
|
73
77
|
const manager = createStoreManager(
|
|
74
78
|
NODES_CLOSEST_TO_DATA[0],
|
|
75
79
|
[NODES_CLOSEST_TO_DATA[1], NODES_CLOSEST_TO_DATA[2], NODES_CLOSEST_TO_DATA[3]],
|
|
76
80
|
replicateData,
|
|
77
|
-
|
|
81
|
+
setAllEntriesAsStale
|
|
78
82
|
)
|
|
79
|
-
manager.
|
|
83
|
+
manager.onContactAdded({ nodeId: getRawFromDhtAddress(NODES_CLOSEST_TO_DATA[4]), type: NodeType.NODEJS })
|
|
80
84
|
await wait(50)
|
|
81
85
|
expect(replicateData).not.toHaveBeenCalled()
|
|
82
|
-
expect(
|
|
86
|
+
expect(setAllEntriesAsStale).not.toHaveBeenCalled()
|
|
83
87
|
})
|
|
84
88
|
})
|
|
85
89
|
|
|
@@ -87,48 +91,47 @@ describe('StoreManager', () => {
|
|
|
87
91
|
|
|
88
92
|
it('this node is within redundancy factor', async () => {
|
|
89
93
|
const replicateData = jest.fn<undefined, [ReplicateDataRequest]>()
|
|
90
|
-
const
|
|
94
|
+
const setAllEntriesAsStale = jest.fn<undefined, [DhtAddress]>()
|
|
91
95
|
const manager = createStoreManager(
|
|
92
96
|
NODES_CLOSEST_TO_DATA[1],
|
|
93
97
|
[NODES_CLOSEST_TO_DATA[0], NODES_CLOSEST_TO_DATA[2], NODES_CLOSEST_TO_DATA[3]],
|
|
94
98
|
replicateData,
|
|
95
|
-
|
|
99
|
+
setAllEntriesAsStale
|
|
96
100
|
)
|
|
97
|
-
manager.
|
|
101
|
+
manager.onContactAdded({ nodeId: getRawFromDhtAddress(NODES_CLOSEST_TO_DATA[4]), type: NodeType.NODEJS })
|
|
98
102
|
await wait(50)
|
|
99
103
|
expect(replicateData).not.toHaveBeenCalled()
|
|
100
|
-
expect(setStale).not.toHaveBeenCalled()
|
|
101
104
|
})
|
|
102
105
|
|
|
103
106
|
it('this node is not within redundancy factor', async () => {
|
|
104
107
|
const replicateData = jest.fn<undefined, [ReplicateDataRequest]>()
|
|
105
|
-
const
|
|
108
|
+
const setAllEntriesAsStale = jest.fn<undefined, [DhtAddress]>()
|
|
106
109
|
const manager = createStoreManager(
|
|
107
110
|
NODES_CLOSEST_TO_DATA[3],
|
|
108
111
|
[NODES_CLOSEST_TO_DATA[0], NODES_CLOSEST_TO_DATA[1], NODES_CLOSEST_TO_DATA[2]],
|
|
109
112
|
replicateData,
|
|
110
|
-
|
|
113
|
+
setAllEntriesAsStale
|
|
111
114
|
)
|
|
112
|
-
manager.
|
|
115
|
+
manager.onContactAdded({ nodeId: getRawFromDhtAddress(NODES_CLOSEST_TO_DATA[4]), type: NodeType.NODEJS })
|
|
113
116
|
await wait(50)
|
|
114
117
|
expect(replicateData).not.toHaveBeenCalled()
|
|
115
|
-
expect(
|
|
116
|
-
expect(
|
|
118
|
+
expect(setAllEntriesAsStale).toHaveBeenCalledTimes(1)
|
|
119
|
+
expect(setAllEntriesAsStale).toHaveBeenCalledWith(getDhtAddressFromRaw(DATA_ENTRY.key))
|
|
117
120
|
})
|
|
118
121
|
|
|
119
122
|
it('this node has less than redundancyFactor neighbors', async () => {
|
|
120
123
|
const replicateData = jest.fn<undefined, [ReplicateDataRequest]>()
|
|
121
|
-
const
|
|
124
|
+
const setAllEntriesAsStale = jest.fn<undefined, [DhtAddress]>()
|
|
122
125
|
const manager = createStoreManager(
|
|
123
126
|
NODES_CLOSEST_TO_DATA[3],
|
|
124
127
|
[NODES_CLOSEST_TO_DATA[0], NODES_CLOSEST_TO_DATA[1]],
|
|
125
128
|
replicateData,
|
|
126
|
-
|
|
129
|
+
setAllEntriesAsStale
|
|
127
130
|
)
|
|
128
|
-
manager.
|
|
131
|
+
manager.onContactAdded({ nodeId: getRawFromDhtAddress(NODES_CLOSEST_TO_DATA[4]), type: NodeType.NODEJS })
|
|
129
132
|
await wait(50)
|
|
130
133
|
expect(replicateData).not.toHaveBeenCalled()
|
|
131
|
-
expect(
|
|
134
|
+
expect(setAllEntriesAsStale).toHaveBeenCalledTimes(0)
|
|
132
135
|
})
|
|
133
136
|
})
|
|
134
137
|
})
|