@streamr/dht 100.0.0-testnet-two.2 → 100.0.0-testnet-two.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +5 -5
- package/dist/src/connection/ConnectionManager.d.ts +5 -4
- package/dist/src/connection/ConnectionManager.js +29 -33
- package/dist/src/connection/ConnectionManager.js.map +1 -1
- package/dist/src/connection/simulator/SimulatorConnection.js +21 -22
- package/dist/src/connection/simulator/SimulatorConnection.js.map +1 -1
- package/dist/src/connection/simulator/SimulatorConnector.js +4 -3
- package/dist/src/connection/simulator/SimulatorConnector.js.map +1 -1
- package/dist/src/connection/webrtc/NodeWebrtcConnection.js +11 -8
- package/dist/src/connection/webrtc/NodeWebrtcConnection.js.map +1 -1
- package/dist/src/connection/websocket/WebsocketConnector.js +4 -3
- package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
- package/dist/src/dht/DhtNode.js +5 -6
- package/dist/src/dht/DhtNode.js.map +1 -1
- package/dist/src/dht/DhtNodeRpcRemote.js +4 -4
- package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
- package/dist/src/dht/PeerManager.js +24 -19
- package/dist/src/dht/PeerManager.js.map +1 -1
- package/dist/src/dht/discovery/PeerDiscovery.js +3 -2
- package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +4 -4
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationRpcRemote.js +2 -2
- package/dist/src/dht/recursive-operation/RecursiveOperationRpcRemote.js.map +1 -1
- package/dist/src/dht/routing/RouterRpcRemote.js +4 -2
- package/dist/src/dht/routing/RouterRpcRemote.js.map +1 -1
- package/dist/src/dht/store/LocalDataStore.d.ts +1 -2
- package/dist/src/dht/store/LocalDataStore.js +11 -10
- package/dist/src/dht/store/LocalDataStore.js.map +1 -1
- package/dist/src/dht/store/StoreManager.js +16 -12
- package/dist/src/dht/store/StoreManager.js.map +1 -1
- package/dist/src/helpers/peerIdFromPeerDescriptor.js +0 -2
- package/dist/src/helpers/peerIdFromPeerDescriptor.js.map +1 -1
- package/package.json +5 -5
- package/src/connection/ConnectionManager.ts +32 -39
- package/src/connection/simulator/SimulatorConnection.ts +21 -23
- package/src/connection/simulator/SimulatorConnector.ts +4 -3
- package/src/connection/webrtc/NodeWebrtcConnection.ts +11 -10
- package/src/connection/websocket/WebsocketConnector.ts +4 -3
- package/src/dht/DhtNode.ts +5 -6
- package/src/dht/DhtNodeRpcRemote.ts +4 -4
- package/src/dht/PeerManager.ts +24 -19
- package/src/dht/discovery/PeerDiscovery.ts +3 -2
- package/src/dht/recursive-operation/RecursiveOperationManager.ts +4 -4
- package/src/dht/recursive-operation/RecursiveOperationRpcRemote.ts +2 -2
- package/src/dht/routing/RouterRpcRemote.ts +4 -4
- package/src/dht/store/LocalDataStore.ts +10 -11
- package/src/dht/store/StoreManager.ts +16 -12
- package/src/helpers/peerIdFromPeerDescriptor.ts +0 -2
- package/test/integration/ReplicateData.test.ts +19 -13
- package/test/unit/LocalDataStore.test.ts +15 -23
package/src/dht/PeerManager.ts
CHANGED
|
@@ -108,8 +108,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
108
108
|
})
|
|
109
109
|
sortingList.addContacts(oldContacts)
|
|
110
110
|
const sortedContacts = sortingList.getAllContacts()
|
|
111
|
-
|
|
112
|
-
this.
|
|
111
|
+
const removableNodeId = sortedContacts[sortedContacts.length - 1].getNodeId()
|
|
112
|
+
this.config.connectionManager?.weakUnlockConnection(removableNodeId)
|
|
113
|
+
this.bucket.remove(getRawFromDhtAddress(removableNodeId))
|
|
113
114
|
this.bucket.add(newContact)
|
|
114
115
|
}
|
|
115
116
|
|
|
@@ -117,8 +118,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
117
118
|
if (this.stopped) {
|
|
118
119
|
return
|
|
119
120
|
}
|
|
120
|
-
|
|
121
|
-
|
|
121
|
+
const nodeId = getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())
|
|
122
|
+
this.config.connectionManager?.weakUnlockConnection(nodeId)
|
|
123
|
+
logger.trace(`Removed contact ${nodeId}`)
|
|
122
124
|
if (this.bucket.count() === 0) {
|
|
123
125
|
this.emit('kBucketEmpty')
|
|
124
126
|
}
|
|
@@ -129,25 +131,27 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
129
131
|
return
|
|
130
132
|
}
|
|
131
133
|
if (contact.getNodeId() !== this.config.localNodeId) {
|
|
134
|
+
const peerDescriptor = contact.getPeerDescriptor()
|
|
135
|
+
const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
|
|
132
136
|
// Important to lock here, before the ping result is known
|
|
133
|
-
this.config.connectionManager?.weakLockConnection(
|
|
137
|
+
this.config.connectionManager?.weakLockConnection(nodeId)
|
|
134
138
|
if (this.connections.has(contact.getNodeId())) {
|
|
135
|
-
logger.trace(`Added new contact ${
|
|
139
|
+
logger.trace(`Added new contact ${nodeId}`)
|
|
136
140
|
} else { // open connection by pinging
|
|
137
|
-
logger.trace('starting ping ' +
|
|
141
|
+
logger.trace('starting ping ' + nodeId)
|
|
138
142
|
contact.ping().then((result) => {
|
|
139
143
|
if (result) {
|
|
140
|
-
logger.trace(`Added new contact ${
|
|
144
|
+
logger.trace(`Added new contact ${nodeId}`)
|
|
141
145
|
} else {
|
|
142
|
-
logger.trace('ping failed ' +
|
|
143
|
-
this.config.connectionManager?.weakUnlockConnection(
|
|
144
|
-
this.removeContact(
|
|
146
|
+
logger.trace('ping failed ' + nodeId)
|
|
147
|
+
this.config.connectionManager?.weakUnlockConnection(nodeId)
|
|
148
|
+
this.removeContact(peerDescriptor)
|
|
145
149
|
this.addClosestContactToBucket()
|
|
146
150
|
}
|
|
147
151
|
return
|
|
148
152
|
}).catch((_e) => {
|
|
149
|
-
this.config.connectionManager?.weakUnlockConnection(
|
|
150
|
-
this.removeContact(
|
|
153
|
+
this.config.connectionManager?.weakUnlockConnection(nodeId)
|
|
154
|
+
this.removeContact(peerDescriptor)
|
|
151
155
|
this.addClosestContactToBucket()
|
|
152
156
|
})
|
|
153
157
|
}
|
|
@@ -185,19 +189,20 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
185
189
|
} else {
|
|
186
190
|
logger.trace('new connection not set to connections, there is already a connection with the peer ID')
|
|
187
191
|
}
|
|
188
|
-
logger.trace('connected: ' +
|
|
192
|
+
logger.trace('connected: ' + nodeId + ' ' + this.connections.size)
|
|
189
193
|
}
|
|
190
194
|
|
|
191
195
|
handleDisconnected(peerDescriptor: PeerDescriptor, gracefulLeave: boolean): void {
|
|
192
|
-
|
|
193
|
-
|
|
196
|
+
const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
|
|
197
|
+
logger.trace('disconnected: ' + nodeId)
|
|
198
|
+
this.connections.delete(nodeId)
|
|
194
199
|
if (this.config.isLayer0) {
|
|
195
200
|
this.bucket.remove(peerDescriptor.nodeId)
|
|
196
201
|
if (gracefulLeave === true) {
|
|
197
|
-
logger.trace(
|
|
202
|
+
logger.trace(nodeId + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
|
|
198
203
|
this.removeContact(peerDescriptor)
|
|
199
204
|
} else {
|
|
200
|
-
logger.trace(
|
|
205
|
+
logger.trace(nodeId + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
|
|
201
206
|
}
|
|
202
207
|
}
|
|
203
208
|
}
|
|
@@ -210,8 +215,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
210
215
|
if (this.stopped) {
|
|
211
216
|
return
|
|
212
217
|
}
|
|
213
|
-
logger.trace(`Removing contact ${getNodeIdFromPeerDescriptor(contact)}`)
|
|
214
218
|
const nodeId = getNodeIdFromPeerDescriptor(contact)
|
|
219
|
+
logger.trace(`Removing contact ${nodeId}`)
|
|
215
220
|
this.bucket.remove(getRawFromDhtAddress(nodeId))
|
|
216
221
|
this.contacts.removeContact(nodeId)
|
|
217
222
|
this.randomPeers.removeContact(nodeId)
|
|
@@ -147,13 +147,14 @@ export class PeerDiscovery {
|
|
|
147
147
|
if (this.isStopped()) {
|
|
148
148
|
return
|
|
149
149
|
}
|
|
150
|
+
const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
|
|
150
151
|
const nodes = this.config.peerManager.getClosestNeighborsTo(
|
|
151
|
-
|
|
152
|
+
localNodeId,
|
|
152
153
|
this.config.parallelism
|
|
153
154
|
)
|
|
154
155
|
await Promise.allSettled(
|
|
155
156
|
nodes.map(async (peer: DhtNodeRpcRemote) => {
|
|
156
|
-
const contacts = await peer.getClosestPeers(
|
|
157
|
+
const contacts = await peer.getClosestPeers(localNodeId)
|
|
157
158
|
this.config.peerManager.handleNewPeers(contacts)
|
|
158
159
|
})
|
|
159
160
|
)
|
|
@@ -96,11 +96,11 @@ export class RecursiveOperationManager {
|
|
|
96
96
|
}
|
|
97
97
|
})
|
|
98
98
|
if (this.config.connections.size === 0) {
|
|
99
|
-
const
|
|
99
|
+
const dataEntries = Array.from(this.config.localDataStore.values(targetId))
|
|
100
100
|
session.onResponseReceived(
|
|
101
101
|
[this.config.localPeerDescriptor],
|
|
102
102
|
[this.config.localPeerDescriptor],
|
|
103
|
-
|
|
103
|
+
dataEntries,
|
|
104
104
|
true
|
|
105
105
|
)
|
|
106
106
|
return session.getResults()
|
|
@@ -124,7 +124,7 @@ export class RecursiveOperationManager {
|
|
|
124
124
|
await wait(50)
|
|
125
125
|
}
|
|
126
126
|
if (operation === RecursiveOperation.FETCH_DATA) {
|
|
127
|
-
const dataEntries = Array.from(this.config.localDataStore.
|
|
127
|
+
const dataEntries = Array.from(this.config.localDataStore.values(targetId))
|
|
128
128
|
if (dataEntries.length > 0) {
|
|
129
129
|
this.sendResponse([], this.config.localPeerDescriptor, session.getId(), [], dataEntries, true)
|
|
130
130
|
}
|
|
@@ -173,7 +173,7 @@ export class RecursiveOperationManager {
|
|
|
173
173
|
// TODO use config option or named constant?
|
|
174
174
|
const closestPeersToDestination = this.getClosestConnections(targetId, 5)
|
|
175
175
|
const dataEntries = (request.operation === RecursiveOperation.FETCH_DATA)
|
|
176
|
-
? Array.from(this.config.localDataStore.
|
|
176
|
+
? Array.from(this.config.localDataStore.values(targetId))
|
|
177
177
|
: []
|
|
178
178
|
if (request.operation === RecursiveOperation.DELETE_DATA) {
|
|
179
179
|
this.config.localDataStore.markAsDeleted(targetId, getNodeIdFromPeerDescriptor(routedMessage.sourcePeer!))
|
|
@@ -33,8 +33,8 @@ export class RecursiveOperationRpcRemote extends RpcRemote<RecursiveOperationRpc
|
|
|
33
33
|
const fromNode = previousPeer
|
|
34
34
|
? getNodeIdFromPeerDescriptor(previousPeer)
|
|
35
35
|
: getNodeIdFromPeerDescriptor(params.sourcePeer!)
|
|
36
|
-
|
|
37
|
-
logger.debug(`Failed to send routeRequest message from ${fromNode} to ${
|
|
36
|
+
const toNode = getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
|
|
37
|
+
logger.debug(`Failed to send routeRequest message from ${fromNode} to ${toNode} with: ${err}`)
|
|
38
38
|
return false
|
|
39
39
|
}
|
|
40
40
|
return true
|
|
@@ -39,7 +39,8 @@ export class RouterRpcRemote extends RpcRemote<RouterRpcClient> {
|
|
|
39
39
|
const fromNode = previousPeer
|
|
40
40
|
? getNodeIdFromPeerDescriptor(previousPeer)
|
|
41
41
|
: getNodeIdFromPeerDescriptor(params.sourcePeer!)
|
|
42
|
-
|
|
42
|
+
const toNode = getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
|
|
43
|
+
logger.trace(`Failed to send routeMessage from ${fromNode} to ${toNode} with: ${err}`)
|
|
43
44
|
return false
|
|
44
45
|
}
|
|
45
46
|
return true
|
|
@@ -67,9 +68,8 @@ export class RouterRpcRemote extends RpcRemote<RouterRpcClient> {
|
|
|
67
68
|
const fromNode = previousPeer
|
|
68
69
|
? getNodeIdFromPeerDescriptor(previousPeer)
|
|
69
70
|
: getNodeIdFromPeerDescriptor(params.sourcePeer!)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
)
|
|
71
|
+
const toNode = getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
|
|
72
|
+
logger.trace(`Failed to send forwardMessage from ${fromNode} to ${toNode} with: ${err}`)
|
|
73
73
|
return false
|
|
74
74
|
}
|
|
75
75
|
return true
|
|
@@ -44,20 +44,19 @@ export class LocalDataStore {
|
|
|
44
44
|
return true
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
public* values(): IterableIterator<DataEntry> {
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
public* values(key?: DhtAddress): IterableIterator<DataEntry> {
|
|
48
|
+
if (key !== undefined) {
|
|
49
|
+
const map = this.store.get(key)
|
|
50
|
+
if (map !== undefined) {
|
|
51
|
+
yield* map.values()
|
|
52
|
+
}
|
|
53
|
+
} else {
|
|
54
|
+
for (const v of this.store.values()) {
|
|
55
|
+
yield* v.values()
|
|
56
|
+
}
|
|
50
57
|
}
|
|
51
58
|
}
|
|
52
59
|
|
|
53
|
-
public getEntries(key: DhtAddress): Map<DhtAddress, DataEntry> {
|
|
54
|
-
const dataEntries = new Map<DhtAddress, DataEntry>
|
|
55
|
-
this.store.get(key)?.forEach((value, creator) => {
|
|
56
|
-
dataEntries.set(creator, value)
|
|
57
|
-
})
|
|
58
|
-
return dataEntries
|
|
59
|
-
}
|
|
60
|
-
|
|
61
60
|
public setStale(key: DhtAddress, creator: DhtAddress, stale: boolean): void {
|
|
62
61
|
const storedEntry = this.store.get(key)?.get(creator)
|
|
63
62
|
if (storedEntry) {
|
|
@@ -62,10 +62,11 @@ export class StoreManager {
|
|
|
62
62
|
|
|
63
63
|
private replicateAndUpdateStaleState(dataEntry: DataEntry, newNode: PeerDescriptor): void {
|
|
64
64
|
const newNodeId = getNodeIdFromPeerDescriptor(newNode)
|
|
65
|
+
const key = getDhtAddressFromRaw(dataEntry.key)
|
|
65
66
|
// TODO use config option or named constant?
|
|
66
|
-
const closestToData = this.config.getClosestNeighborsTo(
|
|
67
|
+
const closestToData = this.config.getClosestNeighborsTo(key, 10)
|
|
67
68
|
const sortedList = new SortedContactList<Contact>({
|
|
68
|
-
referenceId:
|
|
69
|
+
referenceId: key,
|
|
69
70
|
maxSize: 20, // TODO use config option or named constant?
|
|
70
71
|
allowToContainReferenceId: true,
|
|
71
72
|
emitEvents: false
|
|
@@ -89,8 +90,8 @@ export class StoreManager {
|
|
|
89
90
|
await this.replicateDataToContact(dataEntry, newNode)
|
|
90
91
|
})
|
|
91
92
|
}
|
|
92
|
-
} else if (!this.selfIsWithinRedundancyFactor(
|
|
93
|
-
this.config.localDataStore.setStale(
|
|
93
|
+
} else if (!this.selfIsWithinRedundancyFactor(key)) {
|
|
94
|
+
this.config.localDataStore.setStale(key, getDhtAddressFromRaw(dataEntry.creator), true)
|
|
94
95
|
}
|
|
95
96
|
}
|
|
96
97
|
|
|
@@ -111,11 +112,13 @@ export class StoreManager {
|
|
|
111
112
|
const ttl = this.config.highestTtl // ToDo: make TTL decrease according to some nice curve
|
|
112
113
|
const createdAt = Timestamp.now()
|
|
113
114
|
for (let i = 0; i < closestNodes.length && successfulNodes.length < this.config.redundancyFactor; i++) {
|
|
115
|
+
const keyRaw = getRawFromDhtAddress(key)
|
|
116
|
+
const creatorRaw = getRawFromDhtAddress(creator)
|
|
114
117
|
if (areEqualPeerDescriptors(this.config.localPeerDescriptor, closestNodes[i])) {
|
|
115
118
|
this.config.localDataStore.storeEntry({
|
|
116
|
-
key:
|
|
119
|
+
key: keyRaw,
|
|
117
120
|
data,
|
|
118
|
-
creator:
|
|
121
|
+
creator: creatorRaw,
|
|
119
122
|
createdAt,
|
|
120
123
|
storedAt: Timestamp.now(),
|
|
121
124
|
ttl,
|
|
@@ -128,9 +131,9 @@ export class StoreManager {
|
|
|
128
131
|
const rpcRemote = this.config.createRpcRemote(closestNodes[i])
|
|
129
132
|
try {
|
|
130
133
|
await rpcRemote.storeData({
|
|
131
|
-
key:
|
|
134
|
+
key: keyRaw,
|
|
132
135
|
data,
|
|
133
|
-
creator:
|
|
136
|
+
creator: creatorRaw,
|
|
134
137
|
createdAt,
|
|
135
138
|
ttl
|
|
136
139
|
})
|
|
@@ -173,10 +176,11 @@ export class StoreManager {
|
|
|
173
176
|
// sort own contact list according to data id
|
|
174
177
|
const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
|
|
175
178
|
const incomingNodeId = getNodeIdFromPeerDescriptor(incomingPeer)
|
|
179
|
+
const key = getDhtAddressFromRaw(dataEntry.key)
|
|
176
180
|
// TODO use config option or named constant?
|
|
177
|
-
const closestToData = this.config.getClosestNeighborsTo(
|
|
181
|
+
const closestToData = this.config.getClosestNeighborsTo(key, 10)
|
|
178
182
|
const sortedList = new SortedContactList<Contact>({
|
|
179
|
-
referenceId:
|
|
183
|
+
referenceId: key,
|
|
180
184
|
maxSize: this.config.redundancyFactor,
|
|
181
185
|
allowToContainReferenceId: true,
|
|
182
186
|
emitEvents: false
|
|
@@ -192,13 +196,13 @@ export class StoreManager {
|
|
|
192
196
|
// if we are not the closest node to the data, replicate only to the closest one to the data
|
|
193
197
|
: [sortedList.getAllContacts()[0]]
|
|
194
198
|
targets.forEach((contact) => {
|
|
195
|
-
const contactNodeId =
|
|
199
|
+
const contactNodeId = contact.getNodeId()
|
|
196
200
|
if ((incomingNodeId !== contactNodeId) && (localNodeId !== contactNodeId)) {
|
|
197
201
|
setImmediate(() => {
|
|
198
202
|
executeSafePromise(async () => {
|
|
199
203
|
await this.replicateDataToContact(dataEntry, contact.getPeerDescriptor())
|
|
200
204
|
logger.trace('replicateDataToContact() returned', {
|
|
201
|
-
node:
|
|
205
|
+
node: contactNodeId,
|
|
202
206
|
replicateOnlyToClosest: !selfIsPrimaryStorer
|
|
203
207
|
})
|
|
204
208
|
})
|
|
@@ -7,8 +7,6 @@ export const peerIdFromPeerDescriptor = (peerDescriptor: PeerDescriptor): PeerID
|
|
|
7
7
|
return PeerID.fromValue(peerDescriptor.nodeId)
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
-
// TODO could use this in trackerless-network (instead of copy-pasted same implementation)
|
|
11
|
-
// and move this to nodeId.ts
|
|
12
10
|
export const getNodeIdFromPeerDescriptor = (peerDescriptor: PeerDescriptor): DhtAddress => {
|
|
13
11
|
return getDhtAddressFromRaw(peerDescriptor.nodeId)
|
|
14
12
|
}
|
|
@@ -7,11 +7,12 @@ import { execSync } from 'child_process'
|
|
|
7
7
|
import fs from 'fs'
|
|
8
8
|
import { Logger } from '@streamr/utils'
|
|
9
9
|
import { PeerID } from '../../src/helpers/PeerID'
|
|
10
|
-
import {
|
|
10
|
+
import { keyFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../src/helpers/peerIdFromPeerDescriptor'
|
|
11
11
|
import { SortedContactList } from '../../src/dht/contact/SortedContactList'
|
|
12
12
|
import { Contact } from '../../src/dht/contact/Contact'
|
|
13
13
|
import { DhtAddress, getDhtAddressFromRaw, getRawFromDhtAddress } from '../../src/identifiers'
|
|
14
14
|
import { createMockDataEntry } from '../utils/mock/mockDataEntry'
|
|
15
|
+
import { LocalDataStore } from '../../src/dht/store/LocalDataStore'
|
|
15
16
|
|
|
16
17
|
const logger = new Logger(module)
|
|
17
18
|
|
|
@@ -40,6 +41,11 @@ describe('Replicate data from node to node in DHT', () => {
|
|
|
40
41
|
return nodes[Math.floor(Math.random() * nodes.length)]
|
|
41
42
|
}
|
|
42
43
|
*/
|
|
44
|
+
|
|
45
|
+
const getEntries = (key: DhtAddress, localDataStore: LocalDataStore) => {
|
|
46
|
+
return Array.from(localDataStore.values(key))
|
|
47
|
+
}
|
|
48
|
+
|
|
43
49
|
beforeEach(async () => {
|
|
44
50
|
nodes = []
|
|
45
51
|
entryPoint = await createMockConnectionDhtNode(simulator,
|
|
@@ -95,7 +101,7 @@ describe('Replicate data from node to node in DHT', () => {
|
|
|
95
101
|
|
|
96
102
|
logger.info('Nodes sorted according to distance to data are: ')
|
|
97
103
|
closest.forEach((contact) => {
|
|
98
|
-
logger.info(
|
|
104
|
+
logger.info(contact.getNodeId())
|
|
99
105
|
})
|
|
100
106
|
|
|
101
107
|
logger.info('node 0 joining to the DHT')
|
|
@@ -110,17 +116,17 @@ describe('Replicate data from node to node in DHT', () => {
|
|
|
110
116
|
logger.info('Nodes sorted according to distance to data with storing nodes marked are: ')
|
|
111
117
|
|
|
112
118
|
closest.forEach((contact) => {
|
|
113
|
-
const node = nodesById.get(
|
|
119
|
+
const node = nodesById.get(contact.getNodeId())!
|
|
114
120
|
let hasDataMarker = ''
|
|
115
121
|
|
|
116
122
|
// @ts-expect-error private field
|
|
117
123
|
const store = node.localDataStore
|
|
118
|
-
if (
|
|
124
|
+
if (getEntries(dataKey, store).length > 0) {
|
|
119
125
|
hasDataMarker = '<-'
|
|
120
126
|
}
|
|
121
127
|
|
|
122
128
|
// eslint-disable-next-line max-len
|
|
123
|
-
logger.info(peerIdFromPeerDescriptor(contact.getPeerDescriptor()) + ' ' +
|
|
129
|
+
logger.info(peerIdFromPeerDescriptor(contact.getPeerDescriptor()) + ' ' + node.getNodeId() + hasDataMarker)
|
|
124
130
|
})
|
|
125
131
|
|
|
126
132
|
logger.info(NUM_NODES + ' nodes joining layer0 DHT')
|
|
@@ -139,23 +145,23 @@ describe('Replicate data from node to node in DHT', () => {
|
|
|
139
145
|
logger.info('After join of 99 nodes: nodes sorted according to distance to data with storing nodes marked are: ')
|
|
140
146
|
|
|
141
147
|
closest.forEach((contact) => {
|
|
142
|
-
const node = nodesById.get(
|
|
148
|
+
const node = nodesById.get(contact.getNodeId())!
|
|
143
149
|
let hasDataMarker = ''
|
|
144
150
|
|
|
145
151
|
// @ts-expect-error private field
|
|
146
152
|
const store = node.localDataStore
|
|
147
|
-
if (
|
|
153
|
+
if (getEntries(dataKey, store).length > 0) {
|
|
148
154
|
hasDataMarker = '<-'
|
|
149
155
|
}
|
|
150
156
|
|
|
151
|
-
logger.info(
|
|
157
|
+
logger.info(node.getNodeId() + hasDataMarker)
|
|
152
158
|
})
|
|
153
159
|
|
|
154
|
-
const closestNode = nodesById.get(
|
|
160
|
+
const closestNode = nodesById.get(closest[0].getNodeId())!
|
|
155
161
|
|
|
156
162
|
// @ts-expect-error private field
|
|
157
163
|
const store = closestNode.localDataStore
|
|
158
|
-
expect(
|
|
164
|
+
expect(getEntries(dataKey, store).length).toBeGreaterThanOrEqual(1)
|
|
159
165
|
}, 180000)
|
|
160
166
|
|
|
161
167
|
it('Data replicates to the last remaining node if all other nodes leave gracefully', async () => {
|
|
@@ -193,7 +199,7 @@ describe('Replicate data from node to node in DHT', () => {
|
|
|
193
199
|
// @ts-expect-error private field
|
|
194
200
|
const store = nodes[nodeIndex].localDataStore
|
|
195
201
|
logger.info('Stopping node ' + nodeIndex + ' ' +
|
|
196
|
-
(
|
|
202
|
+
((getEntries(dataKey, store).length > 0) ? ', has data' : ' does not have data'))
|
|
197
203
|
|
|
198
204
|
await nodes[nodeIndex].stop()
|
|
199
205
|
}
|
|
@@ -202,8 +208,8 @@ describe('Replicate data from node to node in DHT', () => {
|
|
|
202
208
|
|
|
203
209
|
// @ts-expect-error private field
|
|
204
210
|
const firstStore = nodes[randomIndices[0]].localDataStore
|
|
205
|
-
logger.info('data of ' + randomIndices[0] + ' was ' +
|
|
206
|
-
expect(
|
|
211
|
+
logger.info('data of ' + randomIndices[0] + ' was ' + getEntries(dataKey, firstStore))
|
|
212
|
+
expect(getEntries(dataKey, firstStore).length).toBeGreaterThanOrEqual(1)
|
|
207
213
|
|
|
208
214
|
}, 180000)
|
|
209
215
|
})
|
|
@@ -4,17 +4,13 @@ import {
|
|
|
4
4
|
getNodeIdFromPeerDescriptor,
|
|
5
5
|
} from '../../src/helpers/peerIdFromPeerDescriptor'
|
|
6
6
|
import { createMockPeerDescriptor } from '../utils/utils'
|
|
7
|
-
import { createMockDataEntry
|
|
8
|
-
import {
|
|
7
|
+
import { createMockDataEntry } from '../utils/mock/mockDataEntry'
|
|
8
|
+
import { createRandomDhtAddress, getDhtAddressFromRaw } from '../../src/identifiers'
|
|
9
9
|
|
|
10
10
|
describe('LocalDataStore', () => {
|
|
11
11
|
|
|
12
12
|
let localDataStore: LocalDataStore
|
|
13
13
|
|
|
14
|
-
const getEntryArray = (key: DhtAddress) => {
|
|
15
|
-
return Array.from(localDataStore.getEntries(key).values())
|
|
16
|
-
}
|
|
17
|
-
|
|
18
14
|
beforeEach(() => {
|
|
19
15
|
localDataStore = new LocalDataStore(30 * 1000)
|
|
20
16
|
})
|
|
@@ -26,9 +22,8 @@ describe('LocalDataStore', () => {
|
|
|
26
22
|
it('can store', () => {
|
|
27
23
|
const storedEntry = createMockDataEntry()
|
|
28
24
|
localDataStore.storeEntry(storedEntry)
|
|
29
|
-
const fetchedEntries =
|
|
30
|
-
expect(fetchedEntries).
|
|
31
|
-
expectEqualData(fetchedEntries[0], storedEntry)
|
|
25
|
+
const fetchedEntries = Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))
|
|
26
|
+
expect(fetchedEntries).toIncludeSameMembers([storedEntry])
|
|
32
27
|
})
|
|
33
28
|
|
|
34
29
|
it('multiple storers behind one key', () => {
|
|
@@ -39,10 +34,8 @@ describe('LocalDataStore', () => {
|
|
|
39
34
|
const storedEntry2 = createMockDataEntry({ key, creator: creator2 })
|
|
40
35
|
localDataStore.storeEntry(storedEntry1)
|
|
41
36
|
localDataStore.storeEntry(storedEntry2)
|
|
42
|
-
const fetchedEntries = localDataStore.
|
|
43
|
-
expect(fetchedEntries
|
|
44
|
-
expectEqualData(fetchedEntries.get(creator1)!, storedEntry1)
|
|
45
|
-
expectEqualData(fetchedEntries.get(creator2)!, storedEntry2)
|
|
37
|
+
const fetchedEntries = Array.from(localDataStore.values(key))
|
|
38
|
+
expect(fetchedEntries).toIncludeSameMembers([storedEntry1, storedEntry2])
|
|
46
39
|
})
|
|
47
40
|
|
|
48
41
|
it('can remove data entries', () => {
|
|
@@ -54,9 +47,8 @@ describe('LocalDataStore', () => {
|
|
|
54
47
|
localDataStore.storeEntry(storedEntry1)
|
|
55
48
|
localDataStore.storeEntry(storedEntry2)
|
|
56
49
|
localDataStore.deleteEntry(key, creator1)
|
|
57
|
-
const fetchedEntries =
|
|
58
|
-
expect(fetchedEntries).
|
|
59
|
-
expectEqualData(fetchedEntries[0], storedEntry2)
|
|
50
|
+
const fetchedEntries = Array.from(localDataStore.values(key))
|
|
51
|
+
expect(fetchedEntries).toIncludeSameMembers([storedEntry2])
|
|
60
52
|
})
|
|
61
53
|
|
|
62
54
|
it('can remove all data entries', () => {
|
|
@@ -69,15 +61,15 @@ describe('LocalDataStore', () => {
|
|
|
69
61
|
localDataStore.storeEntry(storedEntry2)
|
|
70
62
|
localDataStore.deleteEntry(key, creator1)
|
|
71
63
|
localDataStore.deleteEntry(key, creator2)
|
|
72
|
-
expect(
|
|
64
|
+
expect(Array.from(localDataStore.values(key))).toHaveLength(0)
|
|
73
65
|
})
|
|
74
66
|
|
|
75
67
|
it('data is deleted after TTL', async () => {
|
|
76
68
|
const storedEntry = createMockDataEntry({ ttl: 1000 })
|
|
77
69
|
localDataStore.storeEntry(storedEntry)
|
|
78
|
-
expect(
|
|
70
|
+
expect(Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))).toHaveLength(1)
|
|
79
71
|
await wait(1100)
|
|
80
|
-
expect(
|
|
72
|
+
expect(Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))).toHaveLength(0)
|
|
81
73
|
})
|
|
82
74
|
|
|
83
75
|
describe('mark data as deleted', () => {
|
|
@@ -86,15 +78,15 @@ describe('LocalDataStore', () => {
|
|
|
86
78
|
const creator1 = createRandomDhtAddress()
|
|
87
79
|
const storedEntry = createMockDataEntry({ creator: creator1 })
|
|
88
80
|
localDataStore.storeEntry(storedEntry)
|
|
89
|
-
const notDeletedData = localDataStore.
|
|
90
|
-
expect(notDeletedData
|
|
81
|
+
const notDeletedData = Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))
|
|
82
|
+
expect(notDeletedData[0]!.deleted).toBeFalse()
|
|
91
83
|
const returnValue = localDataStore.markAsDeleted(
|
|
92
84
|
getDhtAddressFromRaw(storedEntry.key),
|
|
93
85
|
creator1
|
|
94
86
|
)
|
|
95
87
|
expect(returnValue).toBe(true)
|
|
96
|
-
const deletedData = localDataStore.
|
|
97
|
-
expect(deletedData
|
|
88
|
+
const deletedData = Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))
|
|
89
|
+
expect(deletedData[0]!.deleted).toBeTrue()
|
|
98
90
|
})
|
|
99
91
|
|
|
100
92
|
it('data not stored', () => {
|