@streamr/dht 100.1.1 → 100.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +8 -10
- package/dist/src/connection/ConnectionManager.js +4 -6
- package/dist/src/connection/ConnectionManager.js.map +1 -1
- package/dist/src/dht/DhtNode.js +3 -1
- package/dist/src/dht/DhtNode.js.map +1 -1
- package/dist/src/dht/PeerManager.d.ts +4 -3
- package/dist/src/dht/PeerManager.js +54 -58
- package/dist/src/dht/PeerManager.js.map +1 -1
- package/dist/src/dht/contact/ContactList.d.ts +2 -8
- package/dist/src/dht/contact/ContactList.js +1 -10
- package/dist/src/dht/contact/ContactList.js.map +1 -1
- package/dist/src/dht/contact/RandomContactList.js +3 -10
- package/dist/src/dht/contact/RandomContactList.js.map +1 -1
- package/dist/src/dht/contact/RingContactList.d.ts +2 -2
- package/dist/src/dht/contact/RingContactList.js +6 -5
- package/dist/src/dht/contact/RingContactList.js.map +1 -1
- package/dist/src/dht/contact/SortedContactList.d.ts +4 -8
- package/dist/src/dht/contact/SortedContactList.js +24 -51
- package/dist/src/dht/contact/SortedContactList.js.map +1 -1
- package/dist/src/dht/discovery/PeerDiscovery.d.ts +2 -7
- package/dist/src/dht/discovery/PeerDiscovery.js +2 -2
- package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +1 -2
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationSession.js +2 -3
- package/dist/src/dht/recursive-operation/RecursiveOperationSession.js.map +1 -1
- package/dist/src/dht/routing/RoutingSession.js +2 -3
- package/dist/src/dht/routing/RoutingSession.js.map +1 -1
- package/dist/src/dht/routing/RoutingTablesCache.d.ts +1 -1
- package/dist/src/dht/routing/RoutingTablesCache.js.map +1 -1
- package/dist/src/dht/store/StoreManager.js +6 -7
- package/dist/src/dht/store/StoreManager.js.map +1 -1
- package/package.json +8 -10
- package/src/connection/ConnectionManager.ts +4 -6
- package/src/connection/webrtc/BrowserWebrtcConnection.ts +21 -28
- package/src/dht/DhtNode.ts +3 -1
- package/src/dht/PeerManager.ts +54 -58
- package/src/dht/contact/ContactList.ts +2 -12
- package/src/dht/contact/RandomContactList.ts +4 -11
- package/src/dht/contact/RingContactList.ts +7 -5
- package/src/dht/contact/SortedContactList.ts +27 -58
- package/src/dht/discovery/PeerDiscovery.ts +7 -3
- package/src/dht/recursive-operation/RecursiveOperationManager.ts +1 -2
- package/src/dht/recursive-operation/RecursiveOperationSession.ts +2 -3
- package/src/dht/routing/RoutingSession.ts +2 -3
- package/src/dht/routing/RoutingTablesCache.ts +3 -1
- package/src/dht/store/StoreManager.ts +7 -8
- package/test/benchmark/SortedContactListBenchmark.test.ts +9 -52
- package/test/benchmark/hybrid-network-simulation/RingContactList.test.ts +1 -2
- package/test/integration/ReplicateData.test.ts +1 -2
- package/test/unit/PeerManager.test.ts +56 -7
- package/test/unit/SortedContactList.test.ts +17 -36
- package/test/benchmark/kademlia-simulation/Contact.ts +0 -32
- package/test/benchmark/kademlia-simulation/KademliaSimulation.ts +0 -94
- package/test/benchmark/kademlia-simulation/SimulationNode.ts +0 -129
- package/test/data/generateGroundTruthData.ts +0 -71
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Events } from './ContactList'
|
|
2
2
|
import { sortedIndexBy } from 'lodash'
|
|
3
3
|
import EventEmitter from 'eventemitter3'
|
|
4
4
|
import { getDistance } from '../PeerManager'
|
|
@@ -7,9 +7,6 @@ import { DhtAddress, getRawFromDhtAddress } from '../../identifiers'
|
|
|
7
7
|
export interface SortedContactListConfig {
|
|
8
8
|
referenceId: DhtAddress // all contacts in this list are in sorted by the distance to this ID
|
|
9
9
|
allowToContainReferenceId: boolean
|
|
10
|
-
// TODO could maybe optimize this by removing the flag and then we'd check whether we have
|
|
11
|
-
// any listeners before we emit the event
|
|
12
|
-
emitEvents: boolean
|
|
13
10
|
maxSize?: number
|
|
14
11
|
// if set, the list can't contain any contacts which are futher away than this limit
|
|
15
12
|
nodeIdDistanceLimit?: DhtAddress
|
|
@@ -20,7 +17,7 @@ export interface SortedContactListConfig {
|
|
|
20
17
|
export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extends EventEmitter<Events<C>> {
|
|
21
18
|
|
|
22
19
|
private config: SortedContactListConfig
|
|
23
|
-
private contactsById: Map<DhtAddress,
|
|
20
|
+
private contactsById: Map<DhtAddress, C> = new Map()
|
|
24
21
|
private contactIds: DhtAddress[] = []
|
|
25
22
|
|
|
26
23
|
constructor(
|
|
@@ -50,10 +47,10 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
50
47
|
}
|
|
51
48
|
if (!this.contactsById.has(contactId)) {
|
|
52
49
|
if ((this.config.maxSize === undefined) || (this.contactIds.length < this.config.maxSize)) {
|
|
53
|
-
this.contactsById.set(contactId,
|
|
50
|
+
this.contactsById.set(contactId, contact)
|
|
54
51
|
const index = sortedIndexBy(this.contactIds, contactId, (id: DhtAddress) => { return this.distanceToReferenceId(id) })
|
|
55
52
|
this.contactIds.splice(index, 0, contactId)
|
|
56
|
-
if (this.
|
|
53
|
+
if (this.hasEventListeners()) {
|
|
57
54
|
this.emit(
|
|
58
55
|
'contactAdded',
|
|
59
56
|
contact,
|
|
@@ -62,12 +59,12 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
62
59
|
}
|
|
63
60
|
} else if (this.compareIds(this.contactIds[this.config.maxSize - 1], contactId) > 0) {
|
|
64
61
|
const removedId = this.contactIds.pop()
|
|
65
|
-
const removedContact = this.contactsById.get(removedId!)
|
|
62
|
+
const removedContact = this.contactsById.get(removedId!)!
|
|
66
63
|
this.contactsById.delete(removedId!)
|
|
67
|
-
this.contactsById.set(contactId,
|
|
64
|
+
this.contactsById.set(contactId, contact)
|
|
68
65
|
const index = sortedIndexBy(this.contactIds, contactId, (id: DhtAddress) => { return this.distanceToReferenceId(id) })
|
|
69
66
|
this.contactIds.splice(index, 0, contactId)
|
|
70
|
-
if (this.
|
|
67
|
+
if (this.hasEventListeners()) {
|
|
71
68
|
const closestContacts = this.getClosestContacts()
|
|
72
69
|
this.emit(
|
|
73
70
|
'contactRemoved',
|
|
@@ -88,7 +85,7 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
88
85
|
contacts.forEach((contact) => this.addContact(contact))
|
|
89
86
|
}
|
|
90
87
|
|
|
91
|
-
public getContact(id: DhtAddress):
|
|
88
|
+
public getContact(id: DhtAddress): C | undefined {
|
|
92
89
|
return this.contactsById.get(id)
|
|
93
90
|
}
|
|
94
91
|
|
|
@@ -96,52 +93,24 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
96
93
|
return this.contactsById.has(id)
|
|
97
94
|
}
|
|
98
95
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
public setActive(contactId: DhtAddress): void {
|
|
106
|
-
if (this.contactsById.has(contactId)) {
|
|
107
|
-
this.contactsById.get(contactId)!.active = true
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
|
|
96
|
+
/*
|
|
97
|
+
* Closest first then others in ascending distance order
|
|
98
|
+
*/
|
|
111
99
|
public getClosestContacts(limit?: number): C[] {
|
|
112
100
|
const ret = this.getAllContacts()
|
|
113
101
|
return (limit === undefined)
|
|
114
102
|
? ret
|
|
115
|
-
: ret.slice(0, limit)
|
|
103
|
+
: ret.slice(0, Math.max(limit, 0))
|
|
116
104
|
}
|
|
117
105
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
return ret
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
public getActiveContacts(limit?: number): C[] {
|
|
133
|
-
const ret: C[] = []
|
|
134
|
-
this.contactIds.forEach((contactId) => {
|
|
135
|
-
const contact = this.contactsById.get(contactId)!
|
|
136
|
-
if (contact.active) {
|
|
137
|
-
ret.push(contact.contact)
|
|
138
|
-
}
|
|
139
|
-
})
|
|
140
|
-
if (limit !== undefined) {
|
|
141
|
-
return ret.slice(0, limit)
|
|
142
|
-
} else {
|
|
143
|
-
return ret
|
|
144
|
-
}
|
|
106
|
+
/*
|
|
107
|
+
* Furthest first then others in descending distance order
|
|
108
|
+
*/
|
|
109
|
+
getFurthestContacts(limit?: number): C[] {
|
|
110
|
+
const ret = this.getClosestContacts().toReversed()
|
|
111
|
+
return (limit === undefined)
|
|
112
|
+
? ret
|
|
113
|
+
: ret.slice(0, Math.max(limit, 0))
|
|
145
114
|
}
|
|
146
115
|
|
|
147
116
|
public compareIds(id1: DhtAddress, id2: DhtAddress): number {
|
|
@@ -158,12 +127,12 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
158
127
|
|
|
159
128
|
public removeContact(id: DhtAddress): boolean {
|
|
160
129
|
if (this.contactsById.has(id)) {
|
|
161
|
-
const removed = this.contactsById.get(id)
|
|
130
|
+
const removed = this.contactsById.get(id)!
|
|
162
131
|
// TODO use sortedIndexBy?
|
|
163
132
|
const index = this.contactIds.findIndex((nodeId) => (nodeId === id))
|
|
164
133
|
this.contactIds.splice(index, 1)
|
|
165
134
|
this.contactsById.delete(id)
|
|
166
|
-
if (this.
|
|
135
|
+
if (this.hasEventListeners()) {
|
|
167
136
|
this.emit(
|
|
168
137
|
'contactRemoved',
|
|
169
138
|
removed,
|
|
@@ -175,12 +144,8 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
175
144
|
return false
|
|
176
145
|
}
|
|
177
146
|
|
|
178
|
-
public isActive(id: DhtAddress): boolean {
|
|
179
|
-
return this.contactsById.has(id) ? this.contactsById.get(id)!.active : false
|
|
180
|
-
}
|
|
181
|
-
|
|
182
147
|
public getAllContacts(): C[] {
|
|
183
|
-
return this.contactIds.map((nodeId) => this.contactsById.get(nodeId)
|
|
148
|
+
return this.contactIds.map((nodeId) => this.contactsById.get(nodeId)!)
|
|
184
149
|
}
|
|
185
150
|
|
|
186
151
|
public getSize(excludedNodeIds?: Set<DhtAddress>): number {
|
|
@@ -204,4 +169,8 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
204
169
|
this.removeAllListeners()
|
|
205
170
|
this.clear()
|
|
206
171
|
}
|
|
172
|
+
|
|
173
|
+
private hasEventListeners(): boolean {
|
|
174
|
+
return this.eventNames().length > 0
|
|
175
|
+
}
|
|
207
176
|
}
|
|
@@ -60,7 +60,7 @@ export class PeerDiscovery {
|
|
|
60
60
|
)))
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
async joinThroughEntryPoint(
|
|
63
|
+
private async joinThroughEntryPoint(
|
|
64
64
|
entryPointDescriptor: PeerDescriptor,
|
|
65
65
|
// Note that this set is mutated by DiscoverySession
|
|
66
66
|
contactedPeers: Set<DhtAddress>,
|
|
@@ -155,14 +155,18 @@ export class PeerDiscovery {
|
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
157
|
|
|
158
|
-
public async rejoinDht(
|
|
158
|
+
public async rejoinDht(
|
|
159
|
+
entryPoint: PeerDescriptor,
|
|
160
|
+
contactedPeers: Set<DhtAddress> = new Set(),
|
|
161
|
+
distantJoinContactPeers: Set<DhtAddress> = new Set()
|
|
162
|
+
): Promise<void> {
|
|
159
163
|
if (this.isStopped() || this.rejoinOngoing) {
|
|
160
164
|
return
|
|
161
165
|
}
|
|
162
166
|
logger.debug(`Rejoining DHT ${this.config.serviceId}`)
|
|
163
167
|
this.rejoinOngoing = true
|
|
164
168
|
try {
|
|
165
|
-
await this.joinThroughEntryPoint(entryPoint,
|
|
169
|
+
await this.joinThroughEntryPoint(entryPoint, contactedPeers, { enabled: true, contactedPeers: distantJoinContactPeers })
|
|
166
170
|
logger.debug(`Rejoined DHT successfully ${this.config.serviceId}!`)
|
|
167
171
|
} catch (err) {
|
|
168
172
|
logger.warn(`Rejoining DHT ${this.config.serviceId} failed`)
|
|
@@ -222,8 +222,7 @@ export class RecursiveOperationManager {
|
|
|
222
222
|
const closestPeers = new SortedContactList<DhtNodeRpcRemote>({
|
|
223
223
|
referenceId,
|
|
224
224
|
maxSize: limit,
|
|
225
|
-
allowToContainReferenceId: true
|
|
226
|
-
emitEvents: false
|
|
225
|
+
allowToContainReferenceId: true
|
|
227
226
|
})
|
|
228
227
|
closestPeers.addContacts(connectedPeers)
|
|
229
228
|
return closestPeers.getClosestContacts(limit).map((peer) => peer.getPeerDescriptor())
|
|
@@ -53,8 +53,7 @@ export class RecursiveOperationSession extends EventEmitter<RecursiveOperationSe
|
|
|
53
53
|
this.results = new SortedContactList({
|
|
54
54
|
referenceId: config.targetId,
|
|
55
55
|
maxSize: 10, // TODO use config option or named constant?
|
|
56
|
-
allowToContainReferenceId: true
|
|
57
|
-
emitEvents: false
|
|
56
|
+
allowToContainReferenceId: true
|
|
58
57
|
})
|
|
59
58
|
this.rpcCommunicator = new ListeningRpcCommunicator(this.id, config.transport, {
|
|
60
59
|
rpcRequestTimeout: 15000 // TODO use config option or named constant?
|
|
@@ -212,7 +211,7 @@ export class RecursiveOperationSession extends EventEmitter<RecursiveOperationSe
|
|
|
212
211
|
|
|
213
212
|
public getResults(): RecursiveOperationResult {
|
|
214
213
|
return {
|
|
215
|
-
closestNodes: this.results.
|
|
214
|
+
closestNodes: this.results.getClosestContacts().map((contact) => contact.getPeerDescriptor()),
|
|
216
215
|
dataEntries: Array.from(this.foundData.values())
|
|
217
216
|
}
|
|
218
217
|
}
|
|
@@ -172,8 +172,7 @@ export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
|
|
|
172
172
|
referenceId: getDhtAddressFromRaw(this.config.routedMessage.target),
|
|
173
173
|
maxSize: ROUTING_TABLE_MAX_SIZE,
|
|
174
174
|
allowToContainReferenceId: true,
|
|
175
|
-
nodeIdDistanceLimit: previousId
|
|
176
|
-
emitEvents: false
|
|
175
|
+
nodeIdDistanceLimit: previousId
|
|
177
176
|
})
|
|
178
177
|
const contacts = Array.from(this.config.connections.values())
|
|
179
178
|
.map((peer) => new RoutingRemoteContact(
|
|
@@ -184,7 +183,7 @@ export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
|
|
|
184
183
|
routingTable.addContacts(contacts)
|
|
185
184
|
this.config.routingTablesCache.set(targetId, routingTable, previousId)
|
|
186
185
|
}
|
|
187
|
-
return routingTable.
|
|
186
|
+
return routingTable.getClosestContacts()
|
|
188
187
|
.filter((contact) => !this.contactedPeers.has(contact.getNodeId()) && !this.config.excludedNodeIds.has(contact.getNodeId()))
|
|
189
188
|
}
|
|
190
189
|
|
|
@@ -4,7 +4,9 @@ import { RoutingRemoteContact } from './RoutingSession'
|
|
|
4
4
|
import { LRUCache } from 'lru-cache'
|
|
5
5
|
|
|
6
6
|
type RoutingTableID = string
|
|
7
|
-
export type RoutingTable = Pick<
|
|
7
|
+
export type RoutingTable = Pick<
|
|
8
|
+
SortedContactList<RoutingRemoteContact>,
|
|
9
|
+
'getClosestContacts' | 'addContacts' | 'addContact' | 'removeContact' | 'stop'>
|
|
8
10
|
|
|
9
11
|
const createRoutingTableId = (targetId: DhtAddress, previousId?: DhtAddress): RoutingTableID => {
|
|
10
12
|
return targetId + (previousId ? previousId : '')
|
|
@@ -64,8 +64,7 @@ export class StoreManager {
|
|
|
64
64
|
const sortedList = new SortedContactList<Contact>({
|
|
65
65
|
referenceId: key,
|
|
66
66
|
maxSize: this.config.redundancyFactor,
|
|
67
|
-
allowToContainReferenceId: true
|
|
68
|
-
emitEvents: false
|
|
67
|
+
allowToContainReferenceId: true
|
|
69
68
|
})
|
|
70
69
|
sortedList.addContact(new Contact(this.config.localPeerDescriptor))
|
|
71
70
|
closestToData.forEach((neighbor) => {
|
|
@@ -173,20 +172,20 @@ export class StoreManager {
|
|
|
173
172
|
const closestToData = this.config.getClosestNeighborsTo(key, 10)
|
|
174
173
|
const sortedList = new SortedContactList<Contact>({
|
|
175
174
|
referenceId: key,
|
|
176
|
-
maxSize: this.config.redundancyFactor,
|
|
177
|
-
allowToContainReferenceId: true
|
|
178
|
-
emitEvents: false
|
|
175
|
+
maxSize: this.config.redundancyFactor,
|
|
176
|
+
allowToContainReferenceId: true
|
|
179
177
|
})
|
|
180
178
|
sortedList.addContact(new Contact(this.config.localPeerDescriptor))
|
|
181
179
|
closestToData.forEach((neighbor) => {
|
|
182
180
|
sortedList.addContact(new Contact(neighbor))
|
|
183
181
|
})
|
|
184
182
|
const selfIsPrimaryStorer = (sortedList.getClosestContactId() === localNodeId)
|
|
185
|
-
const
|
|
183
|
+
const targetLimit = selfIsPrimaryStorer
|
|
186
184
|
// if we are the closest to the data, replicate to all storageRedundancyFactor nearest
|
|
187
|
-
?
|
|
185
|
+
? undefined
|
|
188
186
|
// if we are not the closest node to the data, replicate only to the closest one to the data
|
|
189
|
-
:
|
|
187
|
+
: 1
|
|
188
|
+
const targets = sortedList.getClosestContacts(targetLimit)
|
|
190
189
|
targets.forEach((contact) => {
|
|
191
190
|
const contactNodeId = contact.getNodeId()
|
|
192
191
|
if ((incomingNodeId !== contactNodeId) && (localNodeId !== contactNodeId)) {
|
|
@@ -39,27 +39,14 @@ describe('SortedContactListBenchmark', () => {
|
|
|
39
39
|
}
|
|
40
40
|
const list = new SortedContactList({
|
|
41
41
|
referenceId: createRandomDhtAddress(),
|
|
42
|
-
allowToContainReferenceId: true
|
|
43
|
-
emitEvents: true
|
|
42
|
+
allowToContainReferenceId: true
|
|
44
43
|
})
|
|
45
44
|
|
|
46
|
-
console.time('SortedContactList.addContact()
|
|
45
|
+
console.time('SortedContactList.addContact()')
|
|
47
46
|
for (let i = 0; i < NUM_ADDS; i++) {
|
|
48
47
|
list.addContact(randomIds[i])
|
|
49
48
|
}
|
|
50
|
-
console.timeEnd('SortedContactList.addContact()
|
|
51
|
-
|
|
52
|
-
const list2 = new SortedContactList({
|
|
53
|
-
referenceId: createRandomDhtAddress(),
|
|
54
|
-
allowToContainReferenceId: true,
|
|
55
|
-
emitEvents: false
|
|
56
|
-
})
|
|
57
|
-
|
|
58
|
-
console.time('SortedContactList.addContact() with emitEvents=false')
|
|
59
|
-
for (let i = 0; i < NUM_ADDS; i++) {
|
|
60
|
-
list2.addContact(randomIds[i])
|
|
61
|
-
}
|
|
62
|
-
console.timeEnd('SortedContactList.addContact() with emitEvents=false')
|
|
49
|
+
console.timeEnd('SortedContactList.addContact()')
|
|
63
50
|
|
|
64
51
|
const kBucket = new KBucket<Item>({ localNodeId: getRawFromDhtAddress(createRandomDhtAddress()) })
|
|
65
52
|
console.time('KBucket.add()')
|
|
@@ -81,61 +68,31 @@ describe('SortedContactListBenchmark', () => {
|
|
|
81
68
|
}
|
|
82
69
|
console.timeEnd('kBucket closest()')
|
|
83
70
|
|
|
84
|
-
console.time('SortedContactList.getClosestContacts()
|
|
85
|
-
for (let i = 0; i < NUM_ADDS; i++) {
|
|
86
|
-
const closest = new SortedContactList<Item>({
|
|
87
|
-
referenceId: createRandomDhtAddress(),
|
|
88
|
-
allowToContainReferenceId: true,
|
|
89
|
-
emitEvents: true
|
|
90
|
-
})
|
|
91
|
-
|
|
92
|
-
const arrayFromBucket = kBucket.toArray()
|
|
93
|
-
arrayFromBucket.forEach((contact) => closest.addContact(contact))
|
|
94
|
-
closest.getClosestContacts(20)
|
|
95
|
-
}
|
|
96
|
-
console.timeEnd('SortedContactList.getClosestContacts() with emitEvents=true')
|
|
97
|
-
|
|
98
|
-
console.time('SortedContactList.getClosestContacts() with emitEvents=false')
|
|
99
|
-
for (let i = 0; i < NUM_ADDS; i++) {
|
|
100
|
-
const closest = new SortedContactList<Item>({
|
|
101
|
-
referenceId: createRandomDhtAddress(),
|
|
102
|
-
allowToContainReferenceId: true,
|
|
103
|
-
emitEvents: false
|
|
104
|
-
})
|
|
105
|
-
|
|
106
|
-
const arrayFromBucket = kBucket.toArray()
|
|
107
|
-
arrayFromBucket.forEach((contact) => closest.addContact(contact))
|
|
108
|
-
closest.getClosestContacts(20)
|
|
109
|
-
}
|
|
110
|
-
console.timeEnd('SortedContactList.getClosestContacts() with emitEvents=false')
|
|
111
|
-
|
|
112
|
-
console.time('SortedContactList.getClosestContacts() with emitEvents=false and lodash')
|
|
71
|
+
console.time('SortedContactList.getClosestContacts()')
|
|
113
72
|
for (let i = 0; i < NUM_ADDS; i++) {
|
|
114
73
|
const closest = new SortedContactList<Item>({
|
|
115
74
|
referenceId: createRandomDhtAddress(),
|
|
116
|
-
allowToContainReferenceId: true
|
|
117
|
-
emitEvents: false
|
|
75
|
+
allowToContainReferenceId: true
|
|
118
76
|
})
|
|
119
77
|
|
|
120
78
|
const arrayFromBucket = kBucket.toArray()
|
|
121
79
|
arrayFromBucket.forEach((contact) => closest.addContact(contact))
|
|
122
80
|
closest.getClosestContacts(20)
|
|
123
81
|
}
|
|
124
|
-
console.timeEnd('SortedContactList.getClosestContacts()
|
|
82
|
+
console.timeEnd('SortedContactList.getClosestContacts()')
|
|
125
83
|
|
|
126
|
-
console.time('SortedContactList.getClosestContacts()
|
|
84
|
+
console.time('SortedContactList.getClosestContacts() and addContacts()')
|
|
127
85
|
for (let i = 0; i < NUM_ADDS; i++) {
|
|
128
86
|
const closest = new SortedContactList<Item>({
|
|
129
87
|
referenceId: createRandomDhtAddress(),
|
|
130
|
-
allowToContainReferenceId: true
|
|
131
|
-
emitEvents: false
|
|
88
|
+
allowToContainReferenceId: true
|
|
132
89
|
})
|
|
133
90
|
|
|
134
91
|
const arrayFromBucket = kBucket.toArray()
|
|
135
92
|
closest.addContacts(arrayFromBucket)
|
|
136
93
|
closest.getClosestContacts(20)
|
|
137
94
|
}
|
|
138
|
-
console.timeEnd('SortedContactList.getClosestContacts()
|
|
95
|
+
console.timeEnd('SortedContactList.getClosestContacts() and addContacts()')
|
|
139
96
|
|
|
140
97
|
const shuffled = shuffleArray(kBucket.toArray())
|
|
141
98
|
console.time('kbucket add and closest')
|
|
@@ -61,8 +61,7 @@ const mockData: Array< [number, string] > = [
|
|
|
61
61
|
const mockNodes: MockNode[] = mockData.map(([region, ipAddress]) => new MockNode(region, ipAddress))
|
|
62
62
|
const referenceNode = mockNodes[5]
|
|
63
63
|
const ringContactList: RingContactList<MockNode> = new RingContactList<MockNode>(
|
|
64
|
-
getRingIdRawFromPeerDescriptor(referenceNode.getPeerDescriptor())
|
|
65
|
-
false
|
|
64
|
+
getRingIdRawFromPeerDescriptor(referenceNode.getPeerDescriptor())
|
|
66
65
|
)
|
|
67
66
|
|
|
68
67
|
mockNodes.forEach((node) => ringContactList.addContact(node))
|
|
@@ -56,8 +56,7 @@ describe('Replicate data from node to node in DHT', () => {
|
|
|
56
56
|
const sortedList = new SortedContactList<DhtNode>({
|
|
57
57
|
referenceId: getDhtAddressFromRaw(DATA.key),
|
|
58
58
|
maxSize: 10000,
|
|
59
|
-
allowToContainReferenceId: true
|
|
60
|
-
emitEvents: false
|
|
59
|
+
allowToContainReferenceId: true
|
|
61
60
|
})
|
|
62
61
|
nodes.forEach((node) => sortedList.addContact(node))
|
|
63
62
|
|
|
@@ -1,18 +1,31 @@
|
|
|
1
|
+
import { waitForCondition } from '@streamr/utils'
|
|
2
|
+
import { range, sample, sampleSize, sortBy, without } from 'lodash'
|
|
3
|
+
import { DhtNodeRpcRemote } from '../../src/dht/DhtNodeRpcRemote'
|
|
1
4
|
import { PeerManager, getDistance } from '../../src/dht/PeerManager'
|
|
5
|
+
import { Contact } from '../../src/dht/contact/Contact'
|
|
6
|
+
import { SortedContactList } from '../../src/dht/contact/SortedContactList'
|
|
2
7
|
import { DhtAddress, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
|
|
3
8
|
import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
|
|
4
|
-
import { range, sample, sampleSize, sortBy, without } from 'lodash'
|
|
5
|
-
import { DhtNodeRpcRemote } from '../../src/dht/DhtNodeRpcRemote'
|
|
6
9
|
import { MockRpcCommunicator } from '../utils/mock/MockRpcCommunicator'
|
|
7
10
|
import { createMockPeerDescriptor } from '../utils/utils'
|
|
8
11
|
|
|
9
|
-
const createPeerManager = (
|
|
10
|
-
|
|
12
|
+
const createPeerManager = (
|
|
13
|
+
nodeIds: DhtAddress[],
|
|
14
|
+
localPeerDescriptor = createMockPeerDescriptor(),
|
|
15
|
+
pingFailures: Set<DhtAddress> = new Set()
|
|
16
|
+
) => {
|
|
11
17
|
const manager = new PeerManager({
|
|
12
|
-
localNodeId: getNodeIdFromPeerDescriptor(
|
|
13
|
-
localPeerDescriptor:
|
|
18
|
+
localNodeId: getNodeIdFromPeerDescriptor(localPeerDescriptor),
|
|
19
|
+
localPeerDescriptor: localPeerDescriptor,
|
|
20
|
+
isLayer0: true,
|
|
14
21
|
createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => {
|
|
15
|
-
|
|
22
|
+
const remote = new class extends DhtNodeRpcRemote {
|
|
23
|
+
// eslint-disable-next-line class-methods-use-this
|
|
24
|
+
async ping(): Promise<boolean> {
|
|
25
|
+
return !pingFailures.has(getNodeIdFromPeerDescriptor(peerDescriptor))
|
|
26
|
+
}
|
|
27
|
+
}(localPeerDescriptor, peerDescriptor, undefined as any, new MockRpcCommunicator())
|
|
28
|
+
return remote
|
|
16
29
|
}
|
|
17
30
|
} as any)
|
|
18
31
|
const contacts = nodeIds.map((n) => ({ nodeId: getRawFromDhtAddress(n), type: NodeType.NODEJS }))
|
|
@@ -22,6 +35,22 @@ const createPeerManager = (nodeIds: DhtAddress[]) => {
|
|
|
22
35
|
return manager
|
|
23
36
|
}
|
|
24
37
|
|
|
38
|
+
const getClosestContact = (contacts: PeerDescriptor[], referenceId: DhtAddress): PeerDescriptor | undefined => {
|
|
39
|
+
const list = new SortedContactList({
|
|
40
|
+
referenceId,
|
|
41
|
+
allowToContainReferenceId: false
|
|
42
|
+
})
|
|
43
|
+
for (const contact of contacts) {
|
|
44
|
+
list.addContact(new Contact(contact))
|
|
45
|
+
}
|
|
46
|
+
const id = list.getClosestContactId()
|
|
47
|
+
if (id !== undefined) {
|
|
48
|
+
return contacts.find((c) => getNodeIdFromPeerDescriptor(c) === id)
|
|
49
|
+
} else {
|
|
50
|
+
return undefined
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
25
54
|
describe('PeerManager', () => {
|
|
26
55
|
|
|
27
56
|
it('getClosestContactsTo', () => {
|
|
@@ -61,4 +90,24 @@ describe('PeerManager', () => {
|
|
|
61
90
|
expect(manager.getContactCount(new Set(sampleSize(nodeIds, 2)))).toBe(8)
|
|
62
91
|
expect(manager.getContactCount(new Set([sample(nodeIds)!, createRandomDhtAddress()]))).toBe(9)
|
|
63
92
|
})
|
|
93
|
+
|
|
94
|
+
it('addContact: ping fails', async () => {
|
|
95
|
+
const localPeerDescriptor = createMockPeerDescriptor()
|
|
96
|
+
const successContacts = range(5).map(() => createMockPeerDescriptor())
|
|
97
|
+
const failureContact = createMockPeerDescriptor()
|
|
98
|
+
const manager = createPeerManager([], localPeerDescriptor, new Set([getNodeIdFromPeerDescriptor(failureContact)]))
|
|
99
|
+
for (const successContact of successContacts) {
|
|
100
|
+
manager.addContact(successContact)
|
|
101
|
+
manager.setContactActive(getNodeIdFromPeerDescriptor(successContact))
|
|
102
|
+
manager.onContactDisconnected(getNodeIdFromPeerDescriptor(successContact), false)
|
|
103
|
+
}
|
|
104
|
+
expect(manager.getNeighborCount()).toBe(0)
|
|
105
|
+
manager.addContact(failureContact)
|
|
106
|
+
const closesSuccessContact = getClosestContact(successContacts, getNodeIdFromPeerDescriptor(localPeerDescriptor))!
|
|
107
|
+
await waitForCondition(() => {
|
|
108
|
+
const neighborNodeIds = manager.getNeighbors().map((n) => getNodeIdFromPeerDescriptor(n))
|
|
109
|
+
return neighborNodeIds.includes(getNodeIdFromPeerDescriptor(closesSuccessContact))
|
|
110
|
+
})
|
|
111
|
+
expect(manager.getNeighbors()).toEqual([closesSuccessContact])
|
|
112
|
+
})
|
|
64
113
|
})
|
|
@@ -16,7 +16,7 @@ describe('SortedContactList', () => {
|
|
|
16
16
|
const item4 = createItem(new Uint8Array([0, 0, 0, 4]))
|
|
17
17
|
|
|
18
18
|
it('compares Ids correctly', async () => {
|
|
19
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 10, allowToContainReferenceId: true
|
|
19
|
+
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 10, allowToContainReferenceId: true })
|
|
20
20
|
expect(list.compareIds(item0.getNodeId(), item0.getNodeId())).toBe(0)
|
|
21
21
|
expect(list.compareIds(item1.getNodeId(), item1.getNodeId())).toBe(0)
|
|
22
22
|
expect(list.compareIds(item0.getNodeId(), item1.getNodeId())).toBe(-1)
|
|
@@ -27,32 +27,8 @@ describe('SortedContactList', () => {
|
|
|
27
27
|
expect(list.compareIds(item1.getNodeId(), item4.getNodeId())).toBe(-3)
|
|
28
28
|
})
|
|
29
29
|
|
|
30
|
-
it('orders itself correctly', async () => {
|
|
31
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 10, allowToContainReferenceId: true, emitEvents: false })
|
|
32
|
-
list.addContact(item3)
|
|
33
|
-
list.addContact(item2)
|
|
34
|
-
list.addContact(item1)
|
|
35
|
-
const contacts = list.getUncontactedContacts(3)
|
|
36
|
-
expect(contacts.length).toEqual(3)
|
|
37
|
-
expect(contacts[0]).toEqual(item1)
|
|
38
|
-
expect(contacts[1]).toEqual(item2)
|
|
39
|
-
expect(contacts[2]).toEqual(item3)
|
|
40
|
-
})
|
|
41
|
-
|
|
42
|
-
it('handles contacted nodes correctly', async () => {
|
|
43
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 10, allowToContainReferenceId: false, emitEvents: false })
|
|
44
|
-
list.addContact(item3)
|
|
45
|
-
list.addContact(item2)
|
|
46
|
-
list.addContact(item1)
|
|
47
|
-
list.setContacted(item2.getNodeId())
|
|
48
|
-
const contacts = list.getUncontactedContacts(3)
|
|
49
|
-
expect(contacts.length).toEqual(2)
|
|
50
|
-
expect(contacts[0]).toEqual(item1)
|
|
51
|
-
expect(contacts[1]).toEqual(item3)
|
|
52
|
-
})
|
|
53
|
-
|
|
54
30
|
it('cannot exceed maxSize', async () => {
|
|
55
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 3, allowToContainReferenceId: false
|
|
31
|
+
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 3, allowToContainReferenceId: false })
|
|
56
32
|
const onContactRemoved = jest.fn()
|
|
57
33
|
list.on('contactRemoved', onContactRemoved)
|
|
58
34
|
list.addContact(item1)
|
|
@@ -67,7 +43,7 @@ describe('SortedContactList', () => {
|
|
|
67
43
|
})
|
|
68
44
|
|
|
69
45
|
it('removing contacts', async () => {
|
|
70
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 8, allowToContainReferenceId: false
|
|
46
|
+
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 8, allowToContainReferenceId: false })
|
|
71
47
|
const onContactRemoved = jest.fn()
|
|
72
48
|
list.on('contactRemoved', onContactRemoved)
|
|
73
49
|
list.removeContact(createRandomDhtAddress())
|
|
@@ -96,31 +72,36 @@ describe('SortedContactList', () => {
|
|
|
96
72
|
const list = new SortedContactList({
|
|
97
73
|
referenceId: item0.getNodeId(),
|
|
98
74
|
maxSize: 8,
|
|
99
|
-
allowToContainReferenceId: false
|
|
100
|
-
emitEvents: false
|
|
75
|
+
allowToContainReferenceId: false
|
|
101
76
|
})
|
|
102
77
|
list.addContact(item1)
|
|
103
78
|
list.addContact(item3)
|
|
104
79
|
list.addContact(item4)
|
|
105
80
|
list.addContact(item2)
|
|
106
81
|
expect(list.getClosestContacts(2)).toEqual([item1, item2])
|
|
82
|
+
expect(list.getClosestContacts(10)).toEqual([item1, item2, item3, item4])
|
|
107
83
|
expect(list.getClosestContacts()).toEqual([item1, item2, item3, item4])
|
|
84
|
+
expect(list.getClosestContacts(-2)).toEqual([])
|
|
108
85
|
})
|
|
109
86
|
|
|
110
|
-
it('get
|
|
111
|
-
const list = new SortedContactList({
|
|
87
|
+
it('get furthest contacts', () => {
|
|
88
|
+
const list = new SortedContactList({
|
|
89
|
+
referenceId: item0.getNodeId(),
|
|
90
|
+
maxSize: 8,
|
|
91
|
+
allowToContainReferenceId: false
|
|
92
|
+
})
|
|
112
93
|
list.addContact(item1)
|
|
113
94
|
list.addContact(item3)
|
|
114
95
|
list.addContact(item4)
|
|
115
96
|
list.addContact(item2)
|
|
116
|
-
list.
|
|
117
|
-
list.
|
|
118
|
-
list.
|
|
119
|
-
expect(list.
|
|
97
|
+
expect(list.getFurthestContacts(2)).toEqual([item4, item3])
|
|
98
|
+
expect(list.getFurthestContacts(10)).toEqual([item4, item3, item2, item1])
|
|
99
|
+
expect(list.getFurthestContacts()).toEqual([item4, item3, item2, item1])
|
|
100
|
+
expect(list.getFurthestContacts(-2)).toEqual([])
|
|
120
101
|
})
|
|
121
102
|
|
|
122
103
|
it('does not emit contactAdded if contact did not fit the structure', () => {
|
|
123
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 2, allowToContainReferenceId: false
|
|
104
|
+
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 2, allowToContainReferenceId: false })
|
|
124
105
|
const onContactAdded = jest.fn()
|
|
125
106
|
list.on('contactAdded', onContactAdded)
|
|
126
107
|
list.addContact(item1)
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import type { SimulationNode } from './SimulationNode'
|
|
2
|
-
import { NodeType, PeerDescriptor } from '../../../src/proto/packages/dht/protos/DhtRpc'
|
|
3
|
-
import { DhtAddress, DhtAddressRaw, getRawFromDhtAddress } from '../../../src/identifiers'
|
|
4
|
-
|
|
5
|
-
export class Contact {
|
|
6
|
-
private static counter = 0
|
|
7
|
-
|
|
8
|
-
public ownId: DhtAddress
|
|
9
|
-
public id: DhtAddressRaw
|
|
10
|
-
public vectorClock = 0
|
|
11
|
-
public dhtNode: SimulationNode | undefined
|
|
12
|
-
|
|
13
|
-
constructor(ownId: DhtAddress, dhtNode?: SimulationNode) {
|
|
14
|
-
this.ownId = ownId
|
|
15
|
-
this.vectorClock = Contact.counter++
|
|
16
|
-
this.dhtNode = dhtNode
|
|
17
|
-
this.id = getRawFromDhtAddress(ownId)
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
getPeerDescriptor(): PeerDescriptor {
|
|
21
|
-
const peerDescriptor: PeerDescriptor = {
|
|
22
|
-
nodeId: getRawFromDhtAddress(this.ownId),
|
|
23
|
-
type: NodeType.NODEJS
|
|
24
|
-
}
|
|
25
|
-
return peerDescriptor
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
getNodeId(): DhtAddress {
|
|
29
|
-
return this.ownId
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
}
|