@streamr/dht 100.1.2 → 100.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +6 -6
- package/dist/src/connection/ConnectionManager.js +4 -6
- package/dist/src/connection/ConnectionManager.js.map +1 -1
- package/dist/src/dht/DhtNode.d.ts +1 -0
- package/dist/src/dht/DhtNode.js +4 -5
- package/dist/src/dht/DhtNode.js.map +1 -1
- package/dist/src/dht/PeerManager.d.ts +4 -3
- package/dist/src/dht/PeerManager.js +60 -60
- package/dist/src/dht/PeerManager.js.map +1 -1
- package/dist/src/dht/contact/ContactList.d.ts +2 -7
- package/dist/src/dht/contact/ContactList.js +1 -9
- package/dist/src/dht/contact/ContactList.js.map +1 -1
- package/dist/src/dht/contact/RandomContactList.js +3 -10
- package/dist/src/dht/contact/RandomContactList.js.map +1 -1
- package/dist/src/dht/contact/RingContactList.d.ts +2 -2
- package/dist/src/dht/contact/RingContactList.js +6 -5
- package/dist/src/dht/contact/RingContactList.js.map +1 -1
- package/dist/src/dht/contact/SortedContactList.d.ts +5 -7
- package/dist/src/dht/contact/SortedContactList.js +24 -35
- package/dist/src/dht/contact/SortedContactList.js.map +1 -1
- package/dist/src/dht/discovery/PeerDiscovery.d.ts +2 -7
- package/dist/src/dht/discovery/PeerDiscovery.js +2 -2
- package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +1 -2
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationSession.js +2 -3
- package/dist/src/dht/recursive-operation/RecursiveOperationSession.js.map +1 -1
- package/dist/src/dht/routing/RoutingSession.js +2 -3
- package/dist/src/dht/routing/RoutingSession.js.map +1 -1
- package/dist/src/dht/routing/RoutingTablesCache.d.ts +1 -1
- package/dist/src/dht/routing/RoutingTablesCache.js.map +1 -1
- package/dist/src/dht/store/StoreManager.js +6 -7
- package/dist/src/dht/store/StoreManager.js.map +1 -1
- package/package.json +6 -6
- package/src/connection/ConnectionManager.ts +4 -6
- package/src/connection/webrtc/BrowserWebrtcConnection.ts +21 -28
- package/src/dht/DhtNode.ts +6 -6
- package/src/dht/PeerManager.ts +60 -60
- package/src/dht/contact/ContactList.ts +2 -12
- package/src/dht/contact/RandomContactList.ts +4 -11
- package/src/dht/contact/RingContactList.ts +7 -5
- package/src/dht/contact/SortedContactList.ts +28 -41
- package/src/dht/discovery/PeerDiscovery.ts +7 -3
- package/src/dht/recursive-operation/RecursiveOperationManager.ts +1 -2
- package/src/dht/recursive-operation/RecursiveOperationSession.ts +2 -3
- package/src/dht/routing/RoutingSession.ts +2 -3
- package/src/dht/routing/RoutingTablesCache.ts +3 -1
- package/src/dht/store/StoreManager.ts +7 -8
- package/test/benchmark/SortedContactListBenchmark.test.ts +9 -52
- package/test/benchmark/hybrid-network-simulation/RingContactList.test.ts +1 -2
- package/test/integration/ReplicateData.test.ts +2 -3
- package/test/unit/PeerManager.test.ts +56 -7
- package/test/unit/SortedContactList.test.ts +21 -16
- package/test/utils/utils.ts +2 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Events } from './ContactList'
|
|
2
2
|
import { sortedIndexBy } from 'lodash'
|
|
3
3
|
import EventEmitter from 'eventemitter3'
|
|
4
4
|
import { getDistance } from '../PeerManager'
|
|
@@ -7,9 +7,6 @@ import { DhtAddress, getRawFromDhtAddress } from '../../identifiers'
|
|
|
7
7
|
export interface SortedContactListConfig {
|
|
8
8
|
referenceId: DhtAddress // all contacts in this list are in sorted by the distance to this ID
|
|
9
9
|
allowToContainReferenceId: boolean
|
|
10
|
-
// TODO could maybe optimize this by removing the flag and then we'd check whether we have
|
|
11
|
-
// any listeners before we emit the event
|
|
12
|
-
emitEvents: boolean
|
|
13
10
|
maxSize?: number
|
|
14
11
|
// if set, the list can't contain any contacts which are futher away than this limit
|
|
15
12
|
nodeIdDistanceLimit?: DhtAddress
|
|
@@ -20,7 +17,7 @@ export interface SortedContactListConfig {
|
|
|
20
17
|
export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extends EventEmitter<Events<C>> {
|
|
21
18
|
|
|
22
19
|
private config: SortedContactListConfig
|
|
23
|
-
private contactsById: Map<DhtAddress,
|
|
20
|
+
private contactsById: Map<DhtAddress, C> = new Map()
|
|
24
21
|
private contactIds: DhtAddress[] = []
|
|
25
22
|
|
|
26
23
|
constructor(
|
|
@@ -50,10 +47,10 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
50
47
|
}
|
|
51
48
|
if (!this.contactsById.has(contactId)) {
|
|
52
49
|
if ((this.config.maxSize === undefined) || (this.contactIds.length < this.config.maxSize)) {
|
|
53
|
-
this.contactsById.set(contactId,
|
|
50
|
+
this.contactsById.set(contactId, contact)
|
|
54
51
|
const index = sortedIndexBy(this.contactIds, contactId, (id: DhtAddress) => { return this.distanceToReferenceId(id) })
|
|
55
52
|
this.contactIds.splice(index, 0, contactId)
|
|
56
|
-
if (this.
|
|
53
|
+
if (this.hasEventListeners()) {
|
|
57
54
|
this.emit(
|
|
58
55
|
'contactAdded',
|
|
59
56
|
contact,
|
|
@@ -62,12 +59,12 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
62
59
|
}
|
|
63
60
|
} else if (this.compareIds(this.contactIds[this.config.maxSize - 1], contactId) > 0) {
|
|
64
61
|
const removedId = this.contactIds.pop()
|
|
65
|
-
const removedContact = this.contactsById.get(removedId!)
|
|
62
|
+
const removedContact = this.contactsById.get(removedId!)!
|
|
66
63
|
this.contactsById.delete(removedId!)
|
|
67
|
-
this.contactsById.set(contactId,
|
|
64
|
+
this.contactsById.set(contactId, contact)
|
|
68
65
|
const index = sortedIndexBy(this.contactIds, contactId, (id: DhtAddress) => { return this.distanceToReferenceId(id) })
|
|
69
66
|
this.contactIds.splice(index, 0, contactId)
|
|
70
|
-
if (this.
|
|
67
|
+
if (this.hasEventListeners()) {
|
|
71
68
|
const closestContacts = this.getClosestContacts()
|
|
72
69
|
this.emit(
|
|
73
70
|
'contactRemoved',
|
|
@@ -88,7 +85,7 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
88
85
|
contacts.forEach((contact) => this.addContact(contact))
|
|
89
86
|
}
|
|
90
87
|
|
|
91
|
-
public getContact(id: DhtAddress):
|
|
88
|
+
public getContact(id: DhtAddress): C | undefined {
|
|
92
89
|
return this.contactsById.get(id)
|
|
93
90
|
}
|
|
94
91
|
|
|
@@ -96,32 +93,22 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
96
93
|
return this.contactsById.has(id)
|
|
97
94
|
}
|
|
98
95
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
96
|
+
/*
|
|
97
|
+
* Closest first then others in ascending distance order
|
|
98
|
+
*/
|
|
99
|
+
public getClosestContacts(limit?: number): C[] {
|
|
100
|
+
const limitedContactIds = (limit === undefined) ? this.contactIds : this.contactIds.slice(0, Math.max(limit, 0))
|
|
101
|
+
return limitedContactIds.map((nodeId) => this.contactsById.get(nodeId)!)
|
|
103
102
|
}
|
|
104
103
|
|
|
105
|
-
|
|
106
|
-
|
|
104
|
+
/*
|
|
105
|
+
* Furthest first then others in descending distance order
|
|
106
|
+
*/
|
|
107
|
+
getFurthestContacts(limit?: number): C[] {
|
|
108
|
+
const ret = [...this.getClosestContacts()].reverse()
|
|
107
109
|
return (limit === undefined)
|
|
108
110
|
? ret
|
|
109
|
-
: ret.slice(0, limit)
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
public getActiveContacts(limit?: number): C[] {
|
|
113
|
-
const ret: C[] = []
|
|
114
|
-
this.contactIds.forEach((contactId) => {
|
|
115
|
-
const contact = this.contactsById.get(contactId)!
|
|
116
|
-
if (contact.active) {
|
|
117
|
-
ret.push(contact.contact)
|
|
118
|
-
}
|
|
119
|
-
})
|
|
120
|
-
if (limit !== undefined) {
|
|
121
|
-
return ret.slice(0, limit)
|
|
122
|
-
} else {
|
|
123
|
-
return ret
|
|
124
|
-
}
|
|
111
|
+
: ret.slice(0, Math.max(limit, 0))
|
|
125
112
|
}
|
|
126
113
|
|
|
127
114
|
public compareIds(id1: DhtAddress, id2: DhtAddress): number {
|
|
@@ -138,12 +125,12 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
138
125
|
|
|
139
126
|
public removeContact(id: DhtAddress): boolean {
|
|
140
127
|
if (this.contactsById.has(id)) {
|
|
141
|
-
const removed = this.contactsById.get(id)
|
|
128
|
+
const removed = this.contactsById.get(id)!
|
|
142
129
|
// TODO use sortedIndexBy?
|
|
143
130
|
const index = this.contactIds.findIndex((nodeId) => (nodeId === id))
|
|
144
131
|
this.contactIds.splice(index, 1)
|
|
145
132
|
this.contactsById.delete(id)
|
|
146
|
-
if (this.
|
|
133
|
+
if (this.hasEventListeners()) {
|
|
147
134
|
this.emit(
|
|
148
135
|
'contactRemoved',
|
|
149
136
|
removed,
|
|
@@ -155,12 +142,8 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
155
142
|
return false
|
|
156
143
|
}
|
|
157
144
|
|
|
158
|
-
public
|
|
159
|
-
return this.contactsById.
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
public getAllContacts(): C[] {
|
|
163
|
-
return this.contactIds.map((nodeId) => this.contactsById.get(nodeId)!.contact)
|
|
145
|
+
public getAllContactsInUndefinedOrder(): IterableIterator<C> {
|
|
146
|
+
return this.contactsById.values()
|
|
164
147
|
}
|
|
165
148
|
|
|
166
149
|
public getSize(excludedNodeIds?: Set<DhtAddress>): number {
|
|
@@ -184,4 +167,8 @@ export class SortedContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
184
167
|
this.removeAllListeners()
|
|
185
168
|
this.clear()
|
|
186
169
|
}
|
|
170
|
+
|
|
171
|
+
private hasEventListeners(): boolean {
|
|
172
|
+
return this.eventNames().length > 0
|
|
173
|
+
}
|
|
187
174
|
}
|
|
@@ -60,7 +60,7 @@ export class PeerDiscovery {
|
|
|
60
60
|
)))
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
async joinThroughEntryPoint(
|
|
63
|
+
private async joinThroughEntryPoint(
|
|
64
64
|
entryPointDescriptor: PeerDescriptor,
|
|
65
65
|
// Note that this set is mutated by DiscoverySession
|
|
66
66
|
contactedPeers: Set<DhtAddress>,
|
|
@@ -155,14 +155,18 @@ export class PeerDiscovery {
|
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
157
|
|
|
158
|
-
public async rejoinDht(
|
|
158
|
+
public async rejoinDht(
|
|
159
|
+
entryPoint: PeerDescriptor,
|
|
160
|
+
contactedPeers: Set<DhtAddress> = new Set(),
|
|
161
|
+
distantJoinContactPeers: Set<DhtAddress> = new Set()
|
|
162
|
+
): Promise<void> {
|
|
159
163
|
if (this.isStopped() || this.rejoinOngoing) {
|
|
160
164
|
return
|
|
161
165
|
}
|
|
162
166
|
logger.debug(`Rejoining DHT ${this.config.serviceId}`)
|
|
163
167
|
this.rejoinOngoing = true
|
|
164
168
|
try {
|
|
165
|
-
await this.joinThroughEntryPoint(entryPoint,
|
|
169
|
+
await this.joinThroughEntryPoint(entryPoint, contactedPeers, { enabled: true, contactedPeers: distantJoinContactPeers })
|
|
166
170
|
logger.debug(`Rejoined DHT successfully ${this.config.serviceId}!`)
|
|
167
171
|
} catch (err) {
|
|
168
172
|
logger.warn(`Rejoining DHT ${this.config.serviceId} failed`)
|
|
@@ -222,8 +222,7 @@ export class RecursiveOperationManager {
|
|
|
222
222
|
const closestPeers = new SortedContactList<DhtNodeRpcRemote>({
|
|
223
223
|
referenceId,
|
|
224
224
|
maxSize: limit,
|
|
225
|
-
allowToContainReferenceId: true
|
|
226
|
-
emitEvents: false
|
|
225
|
+
allowToContainReferenceId: true
|
|
227
226
|
})
|
|
228
227
|
closestPeers.addContacts(connectedPeers)
|
|
229
228
|
return closestPeers.getClosestContacts(limit).map((peer) => peer.getPeerDescriptor())
|
|
@@ -53,8 +53,7 @@ export class RecursiveOperationSession extends EventEmitter<RecursiveOperationSe
|
|
|
53
53
|
this.results = new SortedContactList({
|
|
54
54
|
referenceId: config.targetId,
|
|
55
55
|
maxSize: 10, // TODO use config option or named constant?
|
|
56
|
-
allowToContainReferenceId: true
|
|
57
|
-
emitEvents: false
|
|
56
|
+
allowToContainReferenceId: true
|
|
58
57
|
})
|
|
59
58
|
this.rpcCommunicator = new ListeningRpcCommunicator(this.id, config.transport, {
|
|
60
59
|
rpcRequestTimeout: 15000 // TODO use config option or named constant?
|
|
@@ -212,7 +211,7 @@ export class RecursiveOperationSession extends EventEmitter<RecursiveOperationSe
|
|
|
212
211
|
|
|
213
212
|
public getResults(): RecursiveOperationResult {
|
|
214
213
|
return {
|
|
215
|
-
closestNodes: this.results.
|
|
214
|
+
closestNodes: this.results.getClosestContacts().map((contact) => contact.getPeerDescriptor()),
|
|
216
215
|
dataEntries: Array.from(this.foundData.values())
|
|
217
216
|
}
|
|
218
217
|
}
|
|
@@ -172,8 +172,7 @@ export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
|
|
|
172
172
|
referenceId: getDhtAddressFromRaw(this.config.routedMessage.target),
|
|
173
173
|
maxSize: ROUTING_TABLE_MAX_SIZE,
|
|
174
174
|
allowToContainReferenceId: true,
|
|
175
|
-
nodeIdDistanceLimit: previousId
|
|
176
|
-
emitEvents: false
|
|
175
|
+
nodeIdDistanceLimit: previousId
|
|
177
176
|
})
|
|
178
177
|
const contacts = Array.from(this.config.connections.values())
|
|
179
178
|
.map((peer) => new RoutingRemoteContact(
|
|
@@ -184,7 +183,7 @@ export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
|
|
|
184
183
|
routingTable.addContacts(contacts)
|
|
185
184
|
this.config.routingTablesCache.set(targetId, routingTable, previousId)
|
|
186
185
|
}
|
|
187
|
-
return routingTable.
|
|
186
|
+
return routingTable.getClosestContacts()
|
|
188
187
|
.filter((contact) => !this.contactedPeers.has(contact.getNodeId()) && !this.config.excludedNodeIds.has(contact.getNodeId()))
|
|
189
188
|
}
|
|
190
189
|
|
|
@@ -4,7 +4,9 @@ import { RoutingRemoteContact } from './RoutingSession'
|
|
|
4
4
|
import { LRUCache } from 'lru-cache'
|
|
5
5
|
|
|
6
6
|
type RoutingTableID = string
|
|
7
|
-
export type RoutingTable = Pick<
|
|
7
|
+
export type RoutingTable = Pick<
|
|
8
|
+
SortedContactList<RoutingRemoteContact>,
|
|
9
|
+
'getClosestContacts' | 'addContacts' | 'addContact' | 'removeContact' | 'stop'>
|
|
8
10
|
|
|
9
11
|
const createRoutingTableId = (targetId: DhtAddress, previousId?: DhtAddress): RoutingTableID => {
|
|
10
12
|
return targetId + (previousId ? previousId : '')
|
|
@@ -64,8 +64,7 @@ export class StoreManager {
|
|
|
64
64
|
const sortedList = new SortedContactList<Contact>({
|
|
65
65
|
referenceId: key,
|
|
66
66
|
maxSize: this.config.redundancyFactor,
|
|
67
|
-
allowToContainReferenceId: true
|
|
68
|
-
emitEvents: false
|
|
67
|
+
allowToContainReferenceId: true
|
|
69
68
|
})
|
|
70
69
|
sortedList.addContact(new Contact(this.config.localPeerDescriptor))
|
|
71
70
|
closestToData.forEach((neighbor) => {
|
|
@@ -173,20 +172,20 @@ export class StoreManager {
|
|
|
173
172
|
const closestToData = this.config.getClosestNeighborsTo(key, 10)
|
|
174
173
|
const sortedList = new SortedContactList<Contact>({
|
|
175
174
|
referenceId: key,
|
|
176
|
-
maxSize: this.config.redundancyFactor,
|
|
177
|
-
allowToContainReferenceId: true
|
|
178
|
-
emitEvents: false
|
|
175
|
+
maxSize: this.config.redundancyFactor,
|
|
176
|
+
allowToContainReferenceId: true
|
|
179
177
|
})
|
|
180
178
|
sortedList.addContact(new Contact(this.config.localPeerDescriptor))
|
|
181
179
|
closestToData.forEach((neighbor) => {
|
|
182
180
|
sortedList.addContact(new Contact(neighbor))
|
|
183
181
|
})
|
|
184
182
|
const selfIsPrimaryStorer = (sortedList.getClosestContactId() === localNodeId)
|
|
185
|
-
const
|
|
183
|
+
const targetLimit = selfIsPrimaryStorer
|
|
186
184
|
// if we are the closest to the data, replicate to all storageRedundancyFactor nearest
|
|
187
|
-
?
|
|
185
|
+
? undefined
|
|
188
186
|
// if we are not the closest node to the data, replicate only to the closest one to the data
|
|
189
|
-
:
|
|
187
|
+
: 1
|
|
188
|
+
const targets = sortedList.getClosestContacts(targetLimit)
|
|
190
189
|
targets.forEach((contact) => {
|
|
191
190
|
const contactNodeId = contact.getNodeId()
|
|
192
191
|
if ((incomingNodeId !== contactNodeId) && (localNodeId !== contactNodeId)) {
|
|
@@ -39,27 +39,14 @@ describe('SortedContactListBenchmark', () => {
|
|
|
39
39
|
}
|
|
40
40
|
const list = new SortedContactList({
|
|
41
41
|
referenceId: createRandomDhtAddress(),
|
|
42
|
-
allowToContainReferenceId: true
|
|
43
|
-
emitEvents: true
|
|
42
|
+
allowToContainReferenceId: true
|
|
44
43
|
})
|
|
45
44
|
|
|
46
|
-
console.time('SortedContactList.addContact()
|
|
45
|
+
console.time('SortedContactList.addContact()')
|
|
47
46
|
for (let i = 0; i < NUM_ADDS; i++) {
|
|
48
47
|
list.addContact(randomIds[i])
|
|
49
48
|
}
|
|
50
|
-
console.timeEnd('SortedContactList.addContact()
|
|
51
|
-
|
|
52
|
-
const list2 = new SortedContactList({
|
|
53
|
-
referenceId: createRandomDhtAddress(),
|
|
54
|
-
allowToContainReferenceId: true,
|
|
55
|
-
emitEvents: false
|
|
56
|
-
})
|
|
57
|
-
|
|
58
|
-
console.time('SortedContactList.addContact() with emitEvents=false')
|
|
59
|
-
for (let i = 0; i < NUM_ADDS; i++) {
|
|
60
|
-
list2.addContact(randomIds[i])
|
|
61
|
-
}
|
|
62
|
-
console.timeEnd('SortedContactList.addContact() with emitEvents=false')
|
|
49
|
+
console.timeEnd('SortedContactList.addContact()')
|
|
63
50
|
|
|
64
51
|
const kBucket = new KBucket<Item>({ localNodeId: getRawFromDhtAddress(createRandomDhtAddress()) })
|
|
65
52
|
console.time('KBucket.add()')
|
|
@@ -81,61 +68,31 @@ describe('SortedContactListBenchmark', () => {
|
|
|
81
68
|
}
|
|
82
69
|
console.timeEnd('kBucket closest()')
|
|
83
70
|
|
|
84
|
-
console.time('SortedContactList.getClosestContacts()
|
|
85
|
-
for (let i = 0; i < NUM_ADDS; i++) {
|
|
86
|
-
const closest = new SortedContactList<Item>({
|
|
87
|
-
referenceId: createRandomDhtAddress(),
|
|
88
|
-
allowToContainReferenceId: true,
|
|
89
|
-
emitEvents: true
|
|
90
|
-
})
|
|
91
|
-
|
|
92
|
-
const arrayFromBucket = kBucket.toArray()
|
|
93
|
-
arrayFromBucket.forEach((contact) => closest.addContact(contact))
|
|
94
|
-
closest.getClosestContacts(20)
|
|
95
|
-
}
|
|
96
|
-
console.timeEnd('SortedContactList.getClosestContacts() with emitEvents=true')
|
|
97
|
-
|
|
98
|
-
console.time('SortedContactList.getClosestContacts() with emitEvents=false')
|
|
99
|
-
for (let i = 0; i < NUM_ADDS; i++) {
|
|
100
|
-
const closest = new SortedContactList<Item>({
|
|
101
|
-
referenceId: createRandomDhtAddress(),
|
|
102
|
-
allowToContainReferenceId: true,
|
|
103
|
-
emitEvents: false
|
|
104
|
-
})
|
|
105
|
-
|
|
106
|
-
const arrayFromBucket = kBucket.toArray()
|
|
107
|
-
arrayFromBucket.forEach((contact) => closest.addContact(contact))
|
|
108
|
-
closest.getClosestContacts(20)
|
|
109
|
-
}
|
|
110
|
-
console.timeEnd('SortedContactList.getClosestContacts() with emitEvents=false')
|
|
111
|
-
|
|
112
|
-
console.time('SortedContactList.getClosestContacts() with emitEvents=false and lodash')
|
|
71
|
+
console.time('SortedContactList.getClosestContacts()')
|
|
113
72
|
for (let i = 0; i < NUM_ADDS; i++) {
|
|
114
73
|
const closest = new SortedContactList<Item>({
|
|
115
74
|
referenceId: createRandomDhtAddress(),
|
|
116
|
-
allowToContainReferenceId: true
|
|
117
|
-
emitEvents: false
|
|
75
|
+
allowToContainReferenceId: true
|
|
118
76
|
})
|
|
119
77
|
|
|
120
78
|
const arrayFromBucket = kBucket.toArray()
|
|
121
79
|
arrayFromBucket.forEach((contact) => closest.addContact(contact))
|
|
122
80
|
closest.getClosestContacts(20)
|
|
123
81
|
}
|
|
124
|
-
console.timeEnd('SortedContactList.getClosestContacts()
|
|
82
|
+
console.timeEnd('SortedContactList.getClosestContacts()')
|
|
125
83
|
|
|
126
|
-
console.time('SortedContactList.getClosestContacts()
|
|
84
|
+
console.time('SortedContactList.getClosestContacts() and addContacts()')
|
|
127
85
|
for (let i = 0; i < NUM_ADDS; i++) {
|
|
128
86
|
const closest = new SortedContactList<Item>({
|
|
129
87
|
referenceId: createRandomDhtAddress(),
|
|
130
|
-
allowToContainReferenceId: true
|
|
131
|
-
emitEvents: false
|
|
88
|
+
allowToContainReferenceId: true
|
|
132
89
|
})
|
|
133
90
|
|
|
134
91
|
const arrayFromBucket = kBucket.toArray()
|
|
135
92
|
closest.addContacts(arrayFromBucket)
|
|
136
93
|
closest.getClosestContacts(20)
|
|
137
94
|
}
|
|
138
|
-
console.timeEnd('SortedContactList.getClosestContacts()
|
|
95
|
+
console.timeEnd('SortedContactList.getClosestContacts() and addContacts()')
|
|
139
96
|
|
|
140
97
|
const shuffled = shuffleArray(kBucket.toArray())
|
|
141
98
|
console.time('kbucket add and closest')
|
|
@@ -61,8 +61,7 @@ const mockData: Array< [number, string] > = [
|
|
|
61
61
|
const mockNodes: MockNode[] = mockData.map(([region, ipAddress]) => new MockNode(region, ipAddress))
|
|
62
62
|
const referenceNode = mockNodes[5]
|
|
63
63
|
const ringContactList: RingContactList<MockNode> = new RingContactList<MockNode>(
|
|
64
|
-
getRingIdRawFromPeerDescriptor(referenceNode.getPeerDescriptor())
|
|
65
|
-
false
|
|
64
|
+
getRingIdRawFromPeerDescriptor(referenceNode.getPeerDescriptor())
|
|
66
65
|
)
|
|
67
66
|
|
|
68
67
|
mockNodes.forEach((node) => ringContactList.addContact(node))
|
|
@@ -56,12 +56,11 @@ describe('Replicate data from node to node in DHT', () => {
|
|
|
56
56
|
const sortedList = new SortedContactList<DhtNode>({
|
|
57
57
|
referenceId: getDhtAddressFromRaw(DATA.key),
|
|
58
58
|
maxSize: 10000,
|
|
59
|
-
allowToContainReferenceId: true
|
|
60
|
-
emitEvents: false
|
|
59
|
+
allowToContainReferenceId: true
|
|
61
60
|
})
|
|
62
61
|
nodes.forEach((node) => sortedList.addContact(node))
|
|
63
62
|
|
|
64
|
-
const closest = sortedList.
|
|
63
|
+
const closest = sortedList.getClosestContacts()
|
|
65
64
|
const successfulStorers = await nodes[0].storeDataToDht(getDhtAddressFromRaw(DATA.key), DATA.data!)
|
|
66
65
|
expect(successfulStorers.length).toBe(1)
|
|
67
66
|
|
|
@@ -1,18 +1,31 @@
|
|
|
1
|
+
import { waitForCondition } from '@streamr/utils'
|
|
2
|
+
import { range, sample, sampleSize, sortBy, without } from 'lodash'
|
|
3
|
+
import { DhtNodeRpcRemote } from '../../src/dht/DhtNodeRpcRemote'
|
|
1
4
|
import { PeerManager, getDistance } from '../../src/dht/PeerManager'
|
|
5
|
+
import { Contact } from '../../src/dht/contact/Contact'
|
|
6
|
+
import { SortedContactList } from '../../src/dht/contact/SortedContactList'
|
|
2
7
|
import { DhtAddress, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
|
|
3
8
|
import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
|
|
4
|
-
import { range, sample, sampleSize, sortBy, without } from 'lodash'
|
|
5
|
-
import { DhtNodeRpcRemote } from '../../src/dht/DhtNodeRpcRemote'
|
|
6
9
|
import { MockRpcCommunicator } from '../utils/mock/MockRpcCommunicator'
|
|
7
10
|
import { createMockPeerDescriptor } from '../utils/utils'
|
|
8
11
|
|
|
9
|
-
const createPeerManager = (
|
|
10
|
-
|
|
12
|
+
const createPeerManager = (
|
|
13
|
+
nodeIds: DhtAddress[],
|
|
14
|
+
localPeerDescriptor = createMockPeerDescriptor(),
|
|
15
|
+
pingFailures: Set<DhtAddress> = new Set()
|
|
16
|
+
) => {
|
|
11
17
|
const manager = new PeerManager({
|
|
12
|
-
localNodeId: getNodeIdFromPeerDescriptor(
|
|
13
|
-
localPeerDescriptor:
|
|
18
|
+
localNodeId: getNodeIdFromPeerDescriptor(localPeerDescriptor),
|
|
19
|
+
localPeerDescriptor: localPeerDescriptor,
|
|
20
|
+
isLayer0: true,
|
|
14
21
|
createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => {
|
|
15
|
-
|
|
22
|
+
const remote = new class extends DhtNodeRpcRemote {
|
|
23
|
+
// eslint-disable-next-line class-methods-use-this
|
|
24
|
+
async ping(): Promise<boolean> {
|
|
25
|
+
return !pingFailures.has(getNodeIdFromPeerDescriptor(peerDescriptor))
|
|
26
|
+
}
|
|
27
|
+
}(localPeerDescriptor, peerDescriptor, undefined as any, new MockRpcCommunicator())
|
|
28
|
+
return remote
|
|
16
29
|
}
|
|
17
30
|
} as any)
|
|
18
31
|
const contacts = nodeIds.map((n) => ({ nodeId: getRawFromDhtAddress(n), type: NodeType.NODEJS }))
|
|
@@ -22,6 +35,22 @@ const createPeerManager = (nodeIds: DhtAddress[]) => {
|
|
|
22
35
|
return manager
|
|
23
36
|
}
|
|
24
37
|
|
|
38
|
+
const getClosestContact = (contacts: PeerDescriptor[], referenceId: DhtAddress): PeerDescriptor | undefined => {
|
|
39
|
+
const list = new SortedContactList({
|
|
40
|
+
referenceId,
|
|
41
|
+
allowToContainReferenceId: false
|
|
42
|
+
})
|
|
43
|
+
for (const contact of contacts) {
|
|
44
|
+
list.addContact(new Contact(contact))
|
|
45
|
+
}
|
|
46
|
+
const id = list.getClosestContactId()
|
|
47
|
+
if (id !== undefined) {
|
|
48
|
+
return contacts.find((c) => getNodeIdFromPeerDescriptor(c) === id)
|
|
49
|
+
} else {
|
|
50
|
+
return undefined
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
25
54
|
describe('PeerManager', () => {
|
|
26
55
|
|
|
27
56
|
it('getClosestContactsTo', () => {
|
|
@@ -61,4 +90,24 @@ describe('PeerManager', () => {
|
|
|
61
90
|
expect(manager.getContactCount(new Set(sampleSize(nodeIds, 2)))).toBe(8)
|
|
62
91
|
expect(manager.getContactCount(new Set([sample(nodeIds)!, createRandomDhtAddress()]))).toBe(9)
|
|
63
92
|
})
|
|
93
|
+
|
|
94
|
+
it('addContact: ping fails', async () => {
|
|
95
|
+
const localPeerDescriptor = createMockPeerDescriptor()
|
|
96
|
+
const successContacts = range(5).map(() => createMockPeerDescriptor())
|
|
97
|
+
const failureContact = createMockPeerDescriptor()
|
|
98
|
+
const manager = createPeerManager([], localPeerDescriptor, new Set([getNodeIdFromPeerDescriptor(failureContact)]))
|
|
99
|
+
for (const successContact of successContacts) {
|
|
100
|
+
manager.addContact(successContact)
|
|
101
|
+
manager.setContactActive(getNodeIdFromPeerDescriptor(successContact))
|
|
102
|
+
manager.onContactDisconnected(getNodeIdFromPeerDescriptor(successContact), false)
|
|
103
|
+
}
|
|
104
|
+
expect(manager.getNeighborCount()).toBe(0)
|
|
105
|
+
manager.addContact(failureContact)
|
|
106
|
+
const closesSuccessContact = getClosestContact(successContacts, getNodeIdFromPeerDescriptor(localPeerDescriptor))!
|
|
107
|
+
await waitForCondition(() => {
|
|
108
|
+
const neighborNodeIds = manager.getNeighbors().map((n) => getNodeIdFromPeerDescriptor(n))
|
|
109
|
+
return neighborNodeIds.includes(getNodeIdFromPeerDescriptor(closesSuccessContact))
|
|
110
|
+
})
|
|
111
|
+
expect(manager.getNeighbors()).toEqual([closesSuccessContact])
|
|
112
|
+
})
|
|
64
113
|
})
|
|
@@ -16,7 +16,7 @@ describe('SortedContactList', () => {
|
|
|
16
16
|
const item4 = createItem(new Uint8Array([0, 0, 0, 4]))
|
|
17
17
|
|
|
18
18
|
it('compares Ids correctly', async () => {
|
|
19
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 10, allowToContainReferenceId: true
|
|
19
|
+
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 10, allowToContainReferenceId: true })
|
|
20
20
|
expect(list.compareIds(item0.getNodeId(), item0.getNodeId())).toBe(0)
|
|
21
21
|
expect(list.compareIds(item1.getNodeId(), item1.getNodeId())).toBe(0)
|
|
22
22
|
expect(list.compareIds(item0.getNodeId(), item1.getNodeId())).toBe(-1)
|
|
@@ -28,7 +28,7 @@ describe('SortedContactList', () => {
|
|
|
28
28
|
})
|
|
29
29
|
|
|
30
30
|
it('cannot exceed maxSize', async () => {
|
|
31
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 3, allowToContainReferenceId: false
|
|
31
|
+
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 3, allowToContainReferenceId: false })
|
|
32
32
|
const onContactRemoved = jest.fn()
|
|
33
33
|
list.on('contactRemoved', onContactRemoved)
|
|
34
34
|
list.addContact(item1)
|
|
@@ -36,14 +36,14 @@ describe('SortedContactList', () => {
|
|
|
36
36
|
list.addContact(item3)
|
|
37
37
|
list.addContact(item2)
|
|
38
38
|
expect(list.getSize()).toEqual(3)
|
|
39
|
-
expect(list.
|
|
39
|
+
expect(list.getClosestContacts()).toEqual([item1, item2, item3])
|
|
40
40
|
expect(list.getContactIds()).toEqual([item1.getNodeId(), item2.getNodeId(), item3.getNodeId()])
|
|
41
41
|
expect(onContactRemoved).toBeCalledWith(item4, [item1, item2, item3])
|
|
42
42
|
expect(list.getContact(item4.getNodeId())).toBeFalsy()
|
|
43
43
|
})
|
|
44
44
|
|
|
45
45
|
it('removing contacts', async () => {
|
|
46
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 8, allowToContainReferenceId: false
|
|
46
|
+
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 8, allowToContainReferenceId: false })
|
|
47
47
|
const onContactRemoved = jest.fn()
|
|
48
48
|
list.on('contactRemoved', onContactRemoved)
|
|
49
49
|
list.removeContact(createRandomDhtAddress())
|
|
@@ -57,12 +57,12 @@ describe('SortedContactList', () => {
|
|
|
57
57
|
expect(list.getSize()).toEqual(3)
|
|
58
58
|
expect(list.getContact(item2.getNodeId())).toBeFalsy()
|
|
59
59
|
expect(list.getContactIds()).toEqual(list.getContactIds().sort(list.compareIds))
|
|
60
|
-
expect(list.
|
|
60
|
+
expect(list.getClosestContacts()).toEqual([item1, item3, item4])
|
|
61
61
|
const ret = list.removeContact(getDhtAddressFromRaw(Buffer.from([0, 0, 0, 6])))
|
|
62
62
|
expect(ret).toEqual(false)
|
|
63
63
|
list.removeContact(item3.getNodeId())
|
|
64
64
|
list.removeContact(createRandomDhtAddress())
|
|
65
|
-
expect(list.
|
|
65
|
+
expect(list.getClosestContacts()).toEqual([item1, item4])
|
|
66
66
|
expect(onContactRemoved).toHaveBeenNthCalledWith(1, item3, [])
|
|
67
67
|
expect(onContactRemoved).toHaveBeenNthCalledWith(2, item2, [item1, item3, item4])
|
|
68
68
|
expect(onContactRemoved).toHaveBeenNthCalledWith(3, item3, [item1, item4])
|
|
@@ -72,31 +72,36 @@ describe('SortedContactList', () => {
|
|
|
72
72
|
const list = new SortedContactList({
|
|
73
73
|
referenceId: item0.getNodeId(),
|
|
74
74
|
maxSize: 8,
|
|
75
|
-
allowToContainReferenceId: false
|
|
76
|
-
emitEvents: false
|
|
75
|
+
allowToContainReferenceId: false
|
|
77
76
|
})
|
|
78
77
|
list.addContact(item1)
|
|
79
78
|
list.addContact(item3)
|
|
80
79
|
list.addContact(item4)
|
|
81
80
|
list.addContact(item2)
|
|
82
81
|
expect(list.getClosestContacts(2)).toEqual([item1, item2])
|
|
82
|
+
expect(list.getClosestContacts(10)).toEqual([item1, item2, item3, item4])
|
|
83
83
|
expect(list.getClosestContacts()).toEqual([item1, item2, item3, item4])
|
|
84
|
+
expect(list.getClosestContacts(-2)).toEqual([])
|
|
84
85
|
})
|
|
85
86
|
|
|
86
|
-
it('get
|
|
87
|
-
const list = new SortedContactList({
|
|
87
|
+
it('get furthest contacts', () => {
|
|
88
|
+
const list = new SortedContactList({
|
|
89
|
+
referenceId: item0.getNodeId(),
|
|
90
|
+
maxSize: 8,
|
|
91
|
+
allowToContainReferenceId: false
|
|
92
|
+
})
|
|
88
93
|
list.addContact(item1)
|
|
89
94
|
list.addContact(item3)
|
|
90
95
|
list.addContact(item4)
|
|
91
96
|
list.addContact(item2)
|
|
92
|
-
list.
|
|
93
|
-
list.
|
|
94
|
-
list.
|
|
95
|
-
expect(list.
|
|
97
|
+
expect(list.getFurthestContacts(2)).toEqual([item4, item3])
|
|
98
|
+
expect(list.getFurthestContacts(10)).toEqual([item4, item3, item2, item1])
|
|
99
|
+
expect(list.getFurthestContacts()).toEqual([item4, item3, item2, item1])
|
|
100
|
+
expect(list.getFurthestContacts(-2)).toEqual([])
|
|
96
101
|
})
|
|
97
102
|
|
|
98
103
|
it('does not emit contactAdded if contact did not fit the structure', () => {
|
|
99
|
-
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 2, allowToContainReferenceId: false
|
|
104
|
+
const list = new SortedContactList({ referenceId: item0.getNodeId(), maxSize: 2, allowToContainReferenceId: false })
|
|
100
105
|
const onContactAdded = jest.fn()
|
|
101
106
|
list.on('contactAdded', onContactAdded)
|
|
102
107
|
list.addContact(item1)
|
|
@@ -104,7 +109,7 @@ describe('SortedContactList', () => {
|
|
|
104
109
|
expect(onContactAdded).toBeCalledTimes(2)
|
|
105
110
|
list.addContact(item3)
|
|
106
111
|
expect(onContactAdded).toBeCalledTimes(2)
|
|
107
|
-
expect(list.
|
|
112
|
+
expect(list.getClosestContacts().length).toEqual(2)
|
|
108
113
|
})
|
|
109
114
|
|
|
110
115
|
})
|
package/test/utils/utils.ts
CHANGED
|
@@ -56,6 +56,7 @@ export const createMockRingNode = async (
|
|
|
56
56
|
const opts = {
|
|
57
57
|
peerDescriptor: peerDescriptor,
|
|
58
58
|
transport: mockConnectionManager,
|
|
59
|
+
connectionLocker: mockConnectionManager,
|
|
59
60
|
numberOfNodesPerKBucket: 8,
|
|
60
61
|
maxConnections: maxConnections,
|
|
61
62
|
dhtJoinTimeout,
|
|
@@ -88,6 +89,7 @@ export const createMockConnectionDhtNode = async (
|
|
|
88
89
|
const opts = {
|
|
89
90
|
peerDescriptor: peerDescriptor,
|
|
90
91
|
transport: mockConnectionManager,
|
|
92
|
+
connectionLocker: mockConnectionManager,
|
|
91
93
|
numberOfNodesPerKBucket,
|
|
92
94
|
maxConnections: maxConnections,
|
|
93
95
|
dhtJoinTimeout,
|