@streamr/dht 100.0.0-testnet-two.2 → 100.0.0-testnet-two.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/package.json +5 -5
  2. package/dist/src/connection/ConnectionManager.d.ts +5 -4
  3. package/dist/src/connection/ConnectionManager.js +29 -35
  4. package/dist/src/connection/ConnectionManager.js.map +1 -1
  5. package/dist/src/connection/simulator/SimulatorConnection.js +21 -22
  6. package/dist/src/connection/simulator/SimulatorConnection.js.map +1 -1
  7. package/dist/src/connection/simulator/SimulatorConnector.js +4 -3
  8. package/dist/src/connection/simulator/SimulatorConnector.js.map +1 -1
  9. package/dist/src/connection/webrtc/NodeWebrtcConnection.js +11 -8
  10. package/dist/src/connection/webrtc/NodeWebrtcConnection.js.map +1 -1
  11. package/dist/src/connection/websocket/WebsocketConnector.js +4 -3
  12. package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
  13. package/dist/src/dht/DhtNode.js +5 -6
  14. package/dist/src/dht/DhtNode.js.map +1 -1
  15. package/dist/src/dht/DhtNodeRpcRemote.js +4 -4
  16. package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
  17. package/dist/src/dht/PeerManager.js +24 -19
  18. package/dist/src/dht/PeerManager.js.map +1 -1
  19. package/dist/src/dht/discovery/PeerDiscovery.js +3 -2
  20. package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
  21. package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +4 -4
  22. package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
  23. package/dist/src/dht/recursive-operation/RecursiveOperationRpcRemote.js +2 -2
  24. package/dist/src/dht/recursive-operation/RecursiveOperationRpcRemote.js.map +1 -1
  25. package/dist/src/dht/routing/RouterRpcRemote.js +4 -2
  26. package/dist/src/dht/routing/RouterRpcRemote.js.map +1 -1
  27. package/dist/src/dht/store/LocalDataStore.d.ts +1 -2
  28. package/dist/src/dht/store/LocalDataStore.js +11 -10
  29. package/dist/src/dht/store/LocalDataStore.js.map +1 -1
  30. package/dist/src/dht/store/StoreManager.js +16 -12
  31. package/dist/src/dht/store/StoreManager.js.map +1 -1
  32. package/dist/src/helpers/peerIdFromPeerDescriptor.js +0 -2
  33. package/dist/src/helpers/peerIdFromPeerDescriptor.js.map +1 -1
  34. package/package.json +5 -5
  35. package/src/connection/ConnectionManager.ts +32 -41
  36. package/src/connection/simulator/SimulatorConnection.ts +21 -23
  37. package/src/connection/simulator/SimulatorConnector.ts +4 -3
  38. package/src/connection/webrtc/NodeWebrtcConnection.ts +11 -10
  39. package/src/connection/websocket/WebsocketConnector.ts +4 -3
  40. package/src/dht/DhtNode.ts +5 -6
  41. package/src/dht/DhtNodeRpcRemote.ts +4 -4
  42. package/src/dht/PeerManager.ts +24 -19
  43. package/src/dht/discovery/PeerDiscovery.ts +3 -2
  44. package/src/dht/recursive-operation/RecursiveOperationManager.ts +4 -4
  45. package/src/dht/recursive-operation/RecursiveOperationRpcRemote.ts +2 -2
  46. package/src/dht/routing/RouterRpcRemote.ts +4 -4
  47. package/src/dht/store/LocalDataStore.ts +10 -11
  48. package/src/dht/store/StoreManager.ts +16 -12
  49. package/src/helpers/peerIdFromPeerDescriptor.ts +0 -2
  50. package/test/integration/ReplicateData.test.ts +19 -13
  51. package/test/unit/LocalDataStore.test.ts +15 -23
@@ -41,7 +41,7 @@ export class DhtNodeRpcRemote extends RpcRemote<DhtNodeRpcClient> implements KBu
41
41
  }
42
42
 
43
43
  async getClosestPeers(nodeId: DhtAddress): Promise<PeerDescriptor[]> {
44
- logger.trace(`Requesting getClosestPeers on ${this.serviceId} from ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
44
+ logger.trace(`Requesting getClosestPeers on ${this.serviceId} from ${this.getNodeId()}`)
45
45
  const request: ClosestPeersRequest = {
46
46
  nodeId: getRawFromDhtAddress(nodeId),
47
47
  requestId: v4()
@@ -56,7 +56,7 @@ export class DhtNodeRpcRemote extends RpcRemote<DhtNodeRpcClient> implements KBu
56
56
  }
57
57
 
58
58
  async ping(): Promise<boolean> {
59
- logger.trace(`Requesting ping on ${this.serviceId} from ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
59
+ logger.trace(`Requesting ping on ${this.serviceId} from ${this.getNodeId()}`)
60
60
  const request: PingRequest = {
61
61
  requestId: v4()
62
62
  }
@@ -67,13 +67,13 @@ export class DhtNodeRpcRemote extends RpcRemote<DhtNodeRpcClient> implements KBu
67
67
  return true
68
68
  }
69
69
  } catch (err) {
70
- logger.trace(`ping failed on ${this.serviceId} to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}: ${err}`)
70
+ logger.trace(`ping failed on ${this.serviceId} to ${this.getNodeId()}: ${err}`)
71
71
  }
72
72
  return false
73
73
  }
74
74
 
75
75
  leaveNotice(): void {
76
- logger.trace(`Sending leaveNotice on ${this.serviceId} from ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
76
+ logger.trace(`Sending leaveNotice on ${this.serviceId} from ${this.getNodeId()}`)
77
77
  const options = this.formDhtRpcOptions({
78
78
  notification: true
79
79
  })
@@ -108,8 +108,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
108
108
  })
109
109
  sortingList.addContacts(oldContacts)
110
110
  const sortedContacts = sortingList.getAllContacts()
111
- this.config.connectionManager?.weakUnlockConnection(sortedContacts[sortedContacts.length - 1].getPeerDescriptor())
112
- this.bucket.remove(getRawFromDhtAddress(sortedContacts[sortedContacts.length - 1].getNodeId()))
111
+ const removableNodeId = sortedContacts[sortedContacts.length - 1].getNodeId()
112
+ this.config.connectionManager?.weakUnlockConnection(removableNodeId)
113
+ this.bucket.remove(getRawFromDhtAddress(removableNodeId))
113
114
  this.bucket.add(newContact)
114
115
  }
115
116
 
@@ -117,8 +118,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
117
118
  if (this.stopped) {
118
119
  return
119
120
  }
120
- this.config.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
121
- logger.trace(`Removed contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
121
+ const nodeId = getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())
122
+ this.config.connectionManager?.weakUnlockConnection(nodeId)
123
+ logger.trace(`Removed contact ${nodeId}`)
122
124
  if (this.bucket.count() === 0) {
123
125
  this.emit('kBucketEmpty')
124
126
  }
@@ -129,25 +131,27 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
129
131
  return
130
132
  }
131
133
  if (contact.getNodeId() !== this.config.localNodeId) {
134
+ const peerDescriptor = contact.getPeerDescriptor()
135
+ const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
132
136
  // Important to lock here, before the ping result is known
133
- this.config.connectionManager?.weakLockConnection(contact.getPeerDescriptor())
137
+ this.config.connectionManager?.weakLockConnection(nodeId)
134
138
  if (this.connections.has(contact.getNodeId())) {
135
- logger.trace(`Added new contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
139
+ logger.trace(`Added new contact ${nodeId}`)
136
140
  } else { // open connection by pinging
137
- logger.trace('starting ping ' + getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))
141
+ logger.trace('starting ping ' + nodeId)
138
142
  contact.ping().then((result) => {
139
143
  if (result) {
140
- logger.trace(`Added new contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
144
+ logger.trace(`Added new contact ${nodeId}`)
141
145
  } else {
142
- logger.trace('ping failed ' + getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))
143
- this.config.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
144
- this.removeContact(contact.getPeerDescriptor())
146
+ logger.trace('ping failed ' + nodeId)
147
+ this.config.connectionManager?.weakUnlockConnection(nodeId)
148
+ this.removeContact(peerDescriptor)
145
149
  this.addClosestContactToBucket()
146
150
  }
147
151
  return
148
152
  }).catch((_e) => {
149
- this.config.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
150
- this.removeContact(contact.getPeerDescriptor())
153
+ this.config.connectionManager?.weakUnlockConnection(nodeId)
154
+ this.removeContact(peerDescriptor)
151
155
  this.addClosestContactToBucket()
152
156
  })
153
157
  }
@@ -185,19 +189,20 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
185
189
  } else {
186
190
  logger.trace('new connection not set to connections, there is already a connection with the peer ID')
187
191
  }
188
- logger.trace('connected: ' + getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + this.connections.size)
192
+ logger.trace('connected: ' + nodeId + ' ' + this.connections.size)
189
193
  }
190
194
 
191
195
  handleDisconnected(peerDescriptor: PeerDescriptor, gracefulLeave: boolean): void {
192
- logger.trace('disconnected: ' + getNodeIdFromPeerDescriptor(peerDescriptor))
193
- this.connections.delete(getNodeIdFromPeerDescriptor(peerDescriptor))
196
+ const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
197
+ logger.trace('disconnected: ' + nodeId)
198
+ this.connections.delete(nodeId)
194
199
  if (this.config.isLayer0) {
195
200
  this.bucket.remove(peerDescriptor.nodeId)
196
201
  if (gracefulLeave === true) {
197
- logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
202
+ logger.trace(nodeId + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
198
203
  this.removeContact(peerDescriptor)
199
204
  } else {
200
- logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
205
+ logger.trace(nodeId + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
201
206
  }
202
207
  }
203
208
  }
@@ -210,8 +215,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
210
215
  if (this.stopped) {
211
216
  return
212
217
  }
213
- logger.trace(`Removing contact ${getNodeIdFromPeerDescriptor(contact)}`)
214
218
  const nodeId = getNodeIdFromPeerDescriptor(contact)
219
+ logger.trace(`Removing contact ${nodeId}`)
215
220
  this.bucket.remove(getRawFromDhtAddress(nodeId))
216
221
  this.contacts.removeContact(nodeId)
217
222
  this.randomPeers.removeContact(nodeId)
@@ -147,13 +147,14 @@ export class PeerDiscovery {
147
147
  if (this.isStopped()) {
148
148
  return
149
149
  }
150
+ const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
150
151
  const nodes = this.config.peerManager.getClosestNeighborsTo(
151
- getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor),
152
+ localNodeId,
152
153
  this.config.parallelism
153
154
  )
154
155
  await Promise.allSettled(
155
156
  nodes.map(async (peer: DhtNodeRpcRemote) => {
156
- const contacts = await peer.getClosestPeers(getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor))
157
+ const contacts = await peer.getClosestPeers(localNodeId)
157
158
  this.config.peerManager.handleNewPeers(contacts)
158
159
  })
159
160
  )
@@ -96,11 +96,11 @@ export class RecursiveOperationManager {
96
96
  }
97
97
  })
98
98
  if (this.config.connections.size === 0) {
99
- const data = this.config.localDataStore.getEntries(targetId)
99
+ const dataEntries = Array.from(this.config.localDataStore.values(targetId))
100
100
  session.onResponseReceived(
101
101
  [this.config.localPeerDescriptor],
102
102
  [this.config.localPeerDescriptor],
103
- Array.from(data.values()),
103
+ dataEntries,
104
104
  true
105
105
  )
106
106
  return session.getResults()
@@ -124,7 +124,7 @@ export class RecursiveOperationManager {
124
124
  await wait(50)
125
125
  }
126
126
  if (operation === RecursiveOperation.FETCH_DATA) {
127
- const dataEntries = Array.from(this.config.localDataStore.getEntries(targetId).values())
127
+ const dataEntries = Array.from(this.config.localDataStore.values(targetId))
128
128
  if (dataEntries.length > 0) {
129
129
  this.sendResponse([], this.config.localPeerDescriptor, session.getId(), [], dataEntries, true)
130
130
  }
@@ -173,7 +173,7 @@ export class RecursiveOperationManager {
173
173
  // TODO use config option or named constant?
174
174
  const closestPeersToDestination = this.getClosestConnections(targetId, 5)
175
175
  const dataEntries = (request.operation === RecursiveOperation.FETCH_DATA)
176
- ? Array.from(this.config.localDataStore.getEntries(targetId).values())
176
+ ? Array.from(this.config.localDataStore.values(targetId))
177
177
  : []
178
178
  if (request.operation === RecursiveOperation.DELETE_DATA) {
179
179
  this.config.localDataStore.markAsDeleted(targetId, getNodeIdFromPeerDescriptor(routedMessage.sourcePeer!))
@@ -33,8 +33,8 @@ export class RecursiveOperationRpcRemote extends RpcRemote<RecursiveOperationRpc
33
33
  const fromNode = previousPeer
34
34
  ? getNodeIdFromPeerDescriptor(previousPeer)
35
35
  : getNodeIdFromPeerDescriptor(params.sourcePeer!)
36
- // eslint-disable-next-line max-len
37
- logger.debug(`Failed to send routeRequest message from ${fromNode} to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())} with: ${err}`)
36
+ const toNode = getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
37
+ logger.debug(`Failed to send routeRequest message from ${fromNode} to ${toNode} with: ${err}`)
38
38
  return false
39
39
  }
40
40
  return true
@@ -39,7 +39,8 @@ export class RouterRpcRemote extends RpcRemote<RouterRpcClient> {
39
39
  const fromNode = previousPeer
40
40
  ? getNodeIdFromPeerDescriptor(previousPeer)
41
41
  : getNodeIdFromPeerDescriptor(params.sourcePeer!)
42
- logger.trace(`Failed to send routeMessage from ${fromNode} to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())} with: ${err}`)
42
+ const toNode = getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
43
+ logger.trace(`Failed to send routeMessage from ${fromNode} to ${toNode} with: ${err}`)
43
44
  return false
44
45
  }
45
46
  return true
@@ -67,9 +68,8 @@ export class RouterRpcRemote extends RpcRemote<RouterRpcClient> {
67
68
  const fromNode = previousPeer
68
69
  ? getNodeIdFromPeerDescriptor(previousPeer)
69
70
  : getNodeIdFromPeerDescriptor(params.sourcePeer!)
70
- logger.trace(
71
- `Failed to send forwardMessage from ${fromNode} to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())} with: ${err}`
72
- )
71
+ const toNode = getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
72
+ logger.trace(`Failed to send forwardMessage from ${fromNode} to ${toNode} with: ${err}`)
73
73
  return false
74
74
  }
75
75
  return true
@@ -44,20 +44,19 @@ export class LocalDataStore {
44
44
  return true
45
45
  }
46
46
 
47
- public* values(): IterableIterator<DataEntry> {
48
- for (const v of this.store.values()) {
49
- yield* v.values()
47
+ public* values(key?: DhtAddress): IterableIterator<DataEntry> {
48
+ if (key !== undefined) {
49
+ const map = this.store.get(key)
50
+ if (map !== undefined) {
51
+ yield* map.values()
52
+ }
53
+ } else {
54
+ for (const v of this.store.values()) {
55
+ yield* v.values()
56
+ }
50
57
  }
51
58
  }
52
59
 
53
- public getEntries(key: DhtAddress): Map<DhtAddress, DataEntry> {
54
- const dataEntries = new Map<DhtAddress, DataEntry>
55
- this.store.get(key)?.forEach((value, creator) => {
56
- dataEntries.set(creator, value)
57
- })
58
- return dataEntries
59
- }
60
-
61
60
  public setStale(key: DhtAddress, creator: DhtAddress, stale: boolean): void {
62
61
  const storedEntry = this.store.get(key)?.get(creator)
63
62
  if (storedEntry) {
@@ -62,10 +62,11 @@ export class StoreManager {
62
62
 
63
63
  private replicateAndUpdateStaleState(dataEntry: DataEntry, newNode: PeerDescriptor): void {
64
64
  const newNodeId = getNodeIdFromPeerDescriptor(newNode)
65
+ const key = getDhtAddressFromRaw(dataEntry.key)
65
66
  // TODO use config option or named constant?
66
- const closestToData = this.config.getClosestNeighborsTo(getDhtAddressFromRaw(dataEntry.key), 10)
67
+ const closestToData = this.config.getClosestNeighborsTo(key, 10)
67
68
  const sortedList = new SortedContactList<Contact>({
68
- referenceId: getDhtAddressFromRaw(dataEntry.key),
69
+ referenceId: key,
69
70
  maxSize: 20, // TODO use config option or named constant?
70
71
  allowToContainReferenceId: true,
71
72
  emitEvents: false
@@ -89,8 +90,8 @@ export class StoreManager {
89
90
  await this.replicateDataToContact(dataEntry, newNode)
90
91
  })
91
92
  }
92
- } else if (!this.selfIsWithinRedundancyFactor(getDhtAddressFromRaw(dataEntry.key))) {
93
- this.config.localDataStore.setStale(getDhtAddressFromRaw(dataEntry.key), getDhtAddressFromRaw(dataEntry.creator), true)
93
+ } else if (!this.selfIsWithinRedundancyFactor(key)) {
94
+ this.config.localDataStore.setStale(key, getDhtAddressFromRaw(dataEntry.creator), true)
94
95
  }
95
96
  }
96
97
 
@@ -111,11 +112,13 @@ export class StoreManager {
111
112
  const ttl = this.config.highestTtl // ToDo: make TTL decrease according to some nice curve
112
113
  const createdAt = Timestamp.now()
113
114
  for (let i = 0; i < closestNodes.length && successfulNodes.length < this.config.redundancyFactor; i++) {
115
+ const keyRaw = getRawFromDhtAddress(key)
116
+ const creatorRaw = getRawFromDhtAddress(creator)
114
117
  if (areEqualPeerDescriptors(this.config.localPeerDescriptor, closestNodes[i])) {
115
118
  this.config.localDataStore.storeEntry({
116
- key: getRawFromDhtAddress(key),
119
+ key: keyRaw,
117
120
  data,
118
- creator: getRawFromDhtAddress(creator),
121
+ creator: creatorRaw,
119
122
  createdAt,
120
123
  storedAt: Timestamp.now(),
121
124
  ttl,
@@ -128,9 +131,9 @@ export class StoreManager {
128
131
  const rpcRemote = this.config.createRpcRemote(closestNodes[i])
129
132
  try {
130
133
  await rpcRemote.storeData({
131
- key: getRawFromDhtAddress(key),
134
+ key: keyRaw,
132
135
  data,
133
- creator: getRawFromDhtAddress(creator),
136
+ creator: creatorRaw,
134
137
  createdAt,
135
138
  ttl
136
139
  })
@@ -173,10 +176,11 @@ export class StoreManager {
173
176
  // sort own contact list according to data id
174
177
  const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
175
178
  const incomingNodeId = getNodeIdFromPeerDescriptor(incomingPeer)
179
+ const key = getDhtAddressFromRaw(dataEntry.key)
176
180
  // TODO use config option or named constant?
177
- const closestToData = this.config.getClosestNeighborsTo(getDhtAddressFromRaw(dataEntry.key), 10)
181
+ const closestToData = this.config.getClosestNeighborsTo(key, 10)
178
182
  const sortedList = new SortedContactList<Contact>({
179
- referenceId: getDhtAddressFromRaw(dataEntry.key),
183
+ referenceId: key,
180
184
  maxSize: this.config.redundancyFactor,
181
185
  allowToContainReferenceId: true,
182
186
  emitEvents: false
@@ -192,13 +196,13 @@ export class StoreManager {
192
196
  // if we are not the closest node to the data, replicate only to the closest one to the data
193
197
  : [sortedList.getAllContacts()[0]]
194
198
  targets.forEach((contact) => {
195
- const contactNodeId = getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())
199
+ const contactNodeId = contact.getNodeId()
196
200
  if ((incomingNodeId !== contactNodeId) && (localNodeId !== contactNodeId)) {
197
201
  setImmediate(() => {
198
202
  executeSafePromise(async () => {
199
203
  await this.replicateDataToContact(dataEntry, contact.getPeerDescriptor())
200
204
  logger.trace('replicateDataToContact() returned', {
201
- node: getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()),
205
+ node: contactNodeId,
202
206
  replicateOnlyToClosest: !selfIsPrimaryStorer
203
207
  })
204
208
  })
@@ -7,8 +7,6 @@ export const peerIdFromPeerDescriptor = (peerDescriptor: PeerDescriptor): PeerID
7
7
  return PeerID.fromValue(peerDescriptor.nodeId)
8
8
  }
9
9
 
10
- // TODO could use this in trackerless-network (instead of copy-pasted same implementation)
11
- // and move this to nodeId.ts
12
10
  export const getNodeIdFromPeerDescriptor = (peerDescriptor: PeerDescriptor): DhtAddress => {
13
11
  return getDhtAddressFromRaw(peerDescriptor.nodeId)
14
12
  }
@@ -7,11 +7,12 @@ import { execSync } from 'child_process'
7
7
  import fs from 'fs'
8
8
  import { Logger } from '@streamr/utils'
9
9
  import { PeerID } from '../../src/helpers/PeerID'
10
- import { getNodeIdFromPeerDescriptor, keyFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../src/helpers/peerIdFromPeerDescriptor'
10
+ import { keyFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../src/helpers/peerIdFromPeerDescriptor'
11
11
  import { SortedContactList } from '../../src/dht/contact/SortedContactList'
12
12
  import { Contact } from '../../src/dht/contact/Contact'
13
13
  import { DhtAddress, getDhtAddressFromRaw, getRawFromDhtAddress } from '../../src/identifiers'
14
14
  import { createMockDataEntry } from '../utils/mock/mockDataEntry'
15
+ import { LocalDataStore } from '../../src/dht/store/LocalDataStore'
15
16
 
16
17
  const logger = new Logger(module)
17
18
 
@@ -40,6 +41,11 @@ describe('Replicate data from node to node in DHT', () => {
40
41
  return nodes[Math.floor(Math.random() * nodes.length)]
41
42
  }
42
43
  */
44
+
45
+ const getEntries = (key: DhtAddress, localDataStore: LocalDataStore) => {
46
+ return Array.from(localDataStore.values(key))
47
+ }
48
+
43
49
  beforeEach(async () => {
44
50
  nodes = []
45
51
  entryPoint = await createMockConnectionDhtNode(simulator,
@@ -95,7 +101,7 @@ describe('Replicate data from node to node in DHT', () => {
95
101
 
96
102
  logger.info('Nodes sorted according to distance to data are: ')
97
103
  closest.forEach((contact) => {
98
- logger.info(getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))
104
+ logger.info(contact.getNodeId())
99
105
  })
100
106
 
101
107
  logger.info('node 0 joining to the DHT')
@@ -110,17 +116,17 @@ describe('Replicate data from node to node in DHT', () => {
110
116
  logger.info('Nodes sorted according to distance to data with storing nodes marked are: ')
111
117
 
112
118
  closest.forEach((contact) => {
113
- const node = nodesById.get(getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))!
119
+ const node = nodesById.get(contact.getNodeId())!
114
120
  let hasDataMarker = ''
115
121
 
116
122
  // @ts-expect-error private field
117
123
  const store = node.localDataStore
118
- if (store.getEntries(dataKey)) {
124
+ if (getEntries(dataKey, store).length > 0) {
119
125
  hasDataMarker = '<-'
120
126
  }
121
127
 
122
128
  // eslint-disable-next-line max-len
123
- logger.info(peerIdFromPeerDescriptor(contact.getPeerDescriptor()) + ' ' + getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + hasDataMarker)
129
+ logger.info(peerIdFromPeerDescriptor(contact.getPeerDescriptor()) + ' ' + node.getNodeId() + hasDataMarker)
124
130
  })
125
131
 
126
132
  logger.info(NUM_NODES + ' nodes joining layer0 DHT')
@@ -139,23 +145,23 @@ describe('Replicate data from node to node in DHT', () => {
139
145
  logger.info('After join of 99 nodes: nodes sorted according to distance to data with storing nodes marked are: ')
140
146
 
141
147
  closest.forEach((contact) => {
142
- const node = nodesById.get(getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))!
148
+ const node = nodesById.get(contact.getNodeId())!
143
149
  let hasDataMarker = ''
144
150
 
145
151
  // @ts-expect-error private field
146
152
  const store = node.localDataStore
147
- if (store.getEntries(dataKey)) {
153
+ if (getEntries(dataKey, store).length > 0) {
148
154
  hasDataMarker = '<-'
149
155
  }
150
156
 
151
- logger.info(getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + hasDataMarker)
157
+ logger.info(node.getNodeId() + hasDataMarker)
152
158
  })
153
159
 
154
- const closestNode = nodesById.get(getNodeIdFromPeerDescriptor(closest[0].getPeerDescriptor()))!
160
+ const closestNode = nodesById.get(closest[0].getNodeId())!
155
161
 
156
162
  // @ts-expect-error private field
157
163
  const store = closestNode.localDataStore
158
- expect(store.getEntries(dataKey).size).toBeGreaterThanOrEqual(1)
164
+ expect(getEntries(dataKey, store).length).toBeGreaterThanOrEqual(1)
159
165
  }, 180000)
160
166
 
161
167
  it('Data replicates to the last remaining node if all other nodes leave gracefully', async () => {
@@ -193,7 +199,7 @@ describe('Replicate data from node to node in DHT', () => {
193
199
  // @ts-expect-error private field
194
200
  const store = nodes[nodeIndex].localDataStore
195
201
  logger.info('Stopping node ' + nodeIndex + ' ' +
196
- (store.getEntries(dataKey) ? ', has data' : ' does not have data'))
202
+ ((getEntries(dataKey, store).length > 0) ? ', has data' : ' does not have data'))
197
203
 
198
204
  await nodes[nodeIndex].stop()
199
205
  }
@@ -202,8 +208,8 @@ describe('Replicate data from node to node in DHT', () => {
202
208
 
203
209
  // @ts-expect-error private field
204
210
  const firstStore = nodes[randomIndices[0]].localDataStore
205
- logger.info('data of ' + randomIndices[0] + ' was ' + firstStore.getEntries(dataKey))
206
- expect(firstStore.getEntries(dataKey).size).toBeGreaterThanOrEqual(1)
211
+ logger.info('data of ' + randomIndices[0] + ' was ' + getEntries(dataKey, firstStore))
212
+ expect(getEntries(dataKey, firstStore).length).toBeGreaterThanOrEqual(1)
207
213
 
208
214
  }, 180000)
209
215
  })
@@ -4,17 +4,13 @@ import {
4
4
  getNodeIdFromPeerDescriptor,
5
5
  } from '../../src/helpers/peerIdFromPeerDescriptor'
6
6
  import { createMockPeerDescriptor } from '../utils/utils'
7
- import { createMockDataEntry, expectEqualData } from '../utils/mock/mockDataEntry'
8
- import { DhtAddress, createRandomDhtAddress, getDhtAddressFromRaw } from '../../src/identifiers'
7
+ import { createMockDataEntry } from '../utils/mock/mockDataEntry'
8
+ import { createRandomDhtAddress, getDhtAddressFromRaw } from '../../src/identifiers'
9
9
 
10
10
  describe('LocalDataStore', () => {
11
11
 
12
12
  let localDataStore: LocalDataStore
13
13
 
14
- const getEntryArray = (key: DhtAddress) => {
15
- return Array.from(localDataStore.getEntries(key).values())
16
- }
17
-
18
14
  beforeEach(() => {
19
15
  localDataStore = new LocalDataStore(30 * 1000)
20
16
  })
@@ -26,9 +22,8 @@ describe('LocalDataStore', () => {
26
22
  it('can store', () => {
27
23
  const storedEntry = createMockDataEntry()
28
24
  localDataStore.storeEntry(storedEntry)
29
- const fetchedEntries = getEntryArray(getDhtAddressFromRaw(storedEntry.key))
30
- expect(fetchedEntries).toHaveLength(1)
31
- expectEqualData(fetchedEntries[0], storedEntry)
25
+ const fetchedEntries = Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))
26
+ expect(fetchedEntries).toIncludeSameMembers([storedEntry])
32
27
  })
33
28
 
34
29
  it('multiple storers behind one key', () => {
@@ -39,10 +34,8 @@ describe('LocalDataStore', () => {
39
34
  const storedEntry2 = createMockDataEntry({ key, creator: creator2 })
40
35
  localDataStore.storeEntry(storedEntry1)
41
36
  localDataStore.storeEntry(storedEntry2)
42
- const fetchedEntries = localDataStore.getEntries(key)
43
- expect(fetchedEntries.size).toBe(2)
44
- expectEqualData(fetchedEntries.get(creator1)!, storedEntry1)
45
- expectEqualData(fetchedEntries.get(creator2)!, storedEntry2)
37
+ const fetchedEntries = Array.from(localDataStore.values(key))
38
+ expect(fetchedEntries).toIncludeSameMembers([storedEntry1, storedEntry2])
46
39
  })
47
40
 
48
41
  it('can remove data entries', () => {
@@ -54,9 +47,8 @@ describe('LocalDataStore', () => {
54
47
  localDataStore.storeEntry(storedEntry1)
55
48
  localDataStore.storeEntry(storedEntry2)
56
49
  localDataStore.deleteEntry(key, creator1)
57
- const fetchedEntries = getEntryArray(key)
58
- expect(fetchedEntries).toHaveLength(1)
59
- expectEqualData(fetchedEntries[0], storedEntry2)
50
+ const fetchedEntries = Array.from(localDataStore.values(key))
51
+ expect(fetchedEntries).toIncludeSameMembers([storedEntry2])
60
52
  })
61
53
 
62
54
  it('can remove all data entries', () => {
@@ -69,15 +61,15 @@ describe('LocalDataStore', () => {
69
61
  localDataStore.storeEntry(storedEntry2)
70
62
  localDataStore.deleteEntry(key, creator1)
71
63
  localDataStore.deleteEntry(key, creator2)
72
- expect(getEntryArray(key)).toHaveLength(0)
64
+ expect(Array.from(localDataStore.values(key))).toHaveLength(0)
73
65
  })
74
66
 
75
67
  it('data is deleted after TTL', async () => {
76
68
  const storedEntry = createMockDataEntry({ ttl: 1000 })
77
69
  localDataStore.storeEntry(storedEntry)
78
- expect(getEntryArray(getDhtAddressFromRaw(storedEntry.key))).toHaveLength(1)
70
+ expect(Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))).toHaveLength(1)
79
71
  await wait(1100)
80
- expect(getEntryArray(getDhtAddressFromRaw(storedEntry.key))).toHaveLength(0)
72
+ expect(Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))).toHaveLength(0)
81
73
  })
82
74
 
83
75
  describe('mark data as deleted', () => {
@@ -86,15 +78,15 @@ describe('LocalDataStore', () => {
86
78
  const creator1 = createRandomDhtAddress()
87
79
  const storedEntry = createMockDataEntry({ creator: creator1 })
88
80
  localDataStore.storeEntry(storedEntry)
89
- const notDeletedData = localDataStore.getEntries(getDhtAddressFromRaw(storedEntry.key))
90
- expect(notDeletedData.get(creator1)!.deleted).toBeFalse()
81
+ const notDeletedData = Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))
82
+ expect(notDeletedData[0]!.deleted).toBeFalse()
91
83
  const returnValue = localDataStore.markAsDeleted(
92
84
  getDhtAddressFromRaw(storedEntry.key),
93
85
  creator1
94
86
  )
95
87
  expect(returnValue).toBe(true)
96
- const deletedData = localDataStore.getEntries(getDhtAddressFromRaw(storedEntry.key))
97
- expect(deletedData.get(creator1)!.deleted).toBeTrue()
88
+ const deletedData = Array.from(localDataStore.values(getDhtAddressFromRaw(storedEntry.key)))
89
+ expect(deletedData[0]!.deleted).toBeTrue()
98
90
  })
99
91
 
100
92
  it('data not stored', () => {