@streamr/dht 100.2.4-beta.0 → 100.2.5-beta.0

package/src/dht/discovery/RingDiscoverySession.ts

@@ -1,19 +1,14 @@
-import { Logger, runAndWaitForEvents3 } from '@streamr/utils'
-import EventEmitter from 'eventemitter3'
+import { Gate, Logger, withTimeout } from '@streamr/utils'
 import { v4 } from 'uuid'
+import { DhtAddress, getNodeIdFromPeerDescriptor } from '../../identifiers'
 import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
-import { PeerManager } from '../PeerManager'
 import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
-import { DhtAddress, getNodeIdFromPeerDescriptor } from '../../identifiers'
-import { RingId, RingIdRaw, getLeftDistance, getRingIdFromPeerDescriptor, getRingIdFromRaw } from '../contact/ringIdentifiers'
+import { PeerManager } from '../PeerManager'
 import { RingContacts } from '../contact/RingContactList'
+import { RingId, RingIdRaw, getLeftDistance, getRingIdFromPeerDescriptor, getRingIdFromRaw } from '../contact/ringIdentifiers'
 
 const logger = new Logger(module)
 
-interface RingDiscoverySessionEvents {
-    discoveryCompleted: () => void
-}
-
 interface RingDiscoverySessionConfig {
     targetId: RingIdRaw
     parallelism: number
@@ -21,15 +16,15 @@ interface RingDiscoverySessionConfig {
     peerManager: PeerManager
     // Note that contacted peers will be mutated by the DiscoverySession or other parallel sessions
     contactedPeers: Set<DhtAddress>
+    abortSignal: AbortSignal
 }
 
 export class RingDiscoverySession {
 
     public readonly id = v4()
-    private stopped = false
-    private emitter = new EventEmitter<RingDiscoverySessionEvents>()
     private noProgressCounter = 0
     private ongoingRequests: Set<DhtAddress> = new Set()
+    private doneGate = new Gate(false)
     private readonly config: RingDiscoverySessionConfig
     private numContactedPeers = 0
     private targetIdAsRingId: RingId
@@ -40,7 +35,7 @@ export class RingDiscoverySession {
     }
 
     private addContacts(contacts: PeerDescriptor[]): void {
-        if (this.stopped) {
+        if (this.config.abortSignal.aborted || this.doneGate.isOpen()) {
             return
         }
         for (const contact of contacts) {
@@ -49,7 +44,7 @@ export class RingDiscoverySession {
     }
 
     private async fetchClosestContactsFromRemote(contact: DhtNodeRpcRemote): Promise<RingContacts> {
-        if (this.stopped) {
+        if (this.config.abortSignal.aborted || this.doneGate.isOpen()) {
             return { left: [], right: [] }
         }
         logger.trace(`Getting closest ring peers from contact: ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
@@ -94,7 +89,7 @@ export class RingDiscoverySession {
     }
 
     private findMoreContacts(): void {
-        if (this.stopped) {
+        if (this.config.abortSignal.aborted || this.doneGate.isOpen()) {
             return
         }
         const uncontacted = this.config.peerManager.getClosestRingContactsTo(
@@ -104,8 +99,7 @@ export class RingDiscoverySession {
         )
         if ((uncontacted.left.length === 0 && uncontacted.right.length === 0)
             || this.noProgressCounter >= this.config.noProgressLimit) {
-            this.emitter.emit('discoveryCompleted')
-            this.stopped = true
+            this.doneGate.open()
             return
         }
         // ask from both sides equally
@@ -143,20 +137,12 @@ export class RingDiscoverySession {
     }
 
     public async findClosestNodes(timeout: number): Promise<void> {
-        if (this.config.peerManager.getContactCount(this.config.contactedPeers) === 0) {
+        if (this.config.peerManager.getNearbyContactCount(this.config.contactedPeers) === 0) {
             return
         }
-        // TODO add abortController and signal it in stop()
-        await runAndWaitForEvents3<RingDiscoverySessionEvents>(
-            [this.findMoreContacts.bind(this)],
-            [[this.emitter, 'discoveryCompleted']],
-            timeout
-        )
-    }
-
-    public stop(): void {
-        this.stopped = true
-        this.emitter.emit('discoveryCompleted')
-        this.emitter.removeAllListeners()
+        setImmediate(() => {
+            this.findMoreContacts()
+        })
+        await withTimeout(this.doneGate.waitUntilOpen(), timeout, 'discovery session timed out', this.config.abortSignal)
     }
 }

package/src/dht/recursive-operation/RecursiveOperationManager.ts

@@ -25,11 +25,13 @@ import { ServiceID } from '../../types/ServiceID'
 import { RecursiveOperationRpcLocal } from './RecursiveOperationRpcLocal'
 import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
 import { getDistance } from '../PeerManager'
+import { ConnectionsView } from '../../exports'
 
 interface RecursiveOperationManagerConfig {
     rpcCommunicator: RoutingRpcCommunicator
     sessionTransport: ITransport
     router: Router
+    connectionsView: ConnectionsView
     localPeerDescriptor: PeerDescriptor
     serviceId: ServiceID
     localDataStore: LocalDataStore
@@ -87,14 +89,14 @@ export class RecursiveOperationManager {
             targetId,
             localPeerDescriptor: this.config.localPeerDescriptor,
             // TODO use config option or named constant?
-            waitedRoutingPathCompletions: this.config.sessionTransport.getConnectionCount() > 1 ? 2 : 1,
+            waitedRoutingPathCompletions: this.config.connectionsView.getConnectionCount() > 1 ? 2 : 1,
             operation,
             // TODO would it make sense to give excludedPeer as one of the fields RecursiveOperationSession?
             doRouteRequest: (routedMessage: RouteMessageWrapper) => {
                 return this.doRouteRequest(routedMessage, excludedPeer)
             }
         })
-        if (this.config.sessionTransport.getConnectionCount() === 0) {
+        if (this.config.connectionsView.getConnectionCount() === 0) {
             const dataEntries = Array.from(this.config.localDataStore.values(targetId))
             session.onResponseReceived(
                 getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor),
@@ -218,7 +220,7 @@ export class RecursiveOperationManager {
     }
 
     private getClosestConnectedNodes(referenceId: DhtAddress, limit: number): PeerDescriptor[] {
-        const connectedNodes = this.config.sessionTransport.getConnections().map((c) => this.config.createDhtNodeRpcRemote(c))
+        const connectedNodes = this.config.connectionsView.getConnections().map((c) => this.config.createDhtNodeRpcRemote(c))
         const sorted = new SortedContactList<DhtNodeRpcRemote>({
             referenceId,
             maxSize: limit,

package/src/dht/store/StoreManager.ts

@@ -1,21 +1,28 @@
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { Logger } from '@streamr/utils'
 import {
-    DataEntry, ReplicateDataRequest, PeerDescriptor,
-    StoreDataRequest, StoreDataResponse, RecursiveOperation
-} from '../../proto/packages/dht/protos/DhtRpc'
+    DhtAddress,
+    areEqualPeerDescriptors,
+    getDhtAddressFromRaw,
+    getNodeIdFromPeerDescriptor,
+    getRawFromDhtAddress
+} from '../../identifiers'
 import { Any } from '../../proto/google/protobuf/any'
-import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { Timestamp } from '../../proto/google/protobuf/timestamp'
+import {
+    DataEntry,
+    PeerDescriptor,
+    RecursiveOperation,
+    ReplicateDataRequest,
+    StoreDataRequest, StoreDataResponse
+} from '../../proto/packages/dht/protos/DhtRpc'
 import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
+import { ServiceID } from '../../types/ServiceID'
+import { getClosestNodes } from '../contact/getClosestNodes'
 import { RecursiveOperationManager } from '../recursive-operation/RecursiveOperationManager'
-import { Logger, executeSafePromise } from '@streamr/utils'
 import { LocalDataStore } from './LocalDataStore'
-import { StoreRpcRemote } from './StoreRpcRemote'
-import { Timestamp } from '../../proto/google/protobuf/timestamp'
-import { SortedContactList } from '../contact/SortedContactList'
-import { Contact } from '../contact/Contact'
-import { ServiceID } from '../../types/ServiceID'
-import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
 import { StoreRpcLocal } from './StoreRpcLocal'
-import { getDistance } from '../PeerManager'
+import { StoreRpcRemote } from './StoreRpcRemote'
 
 interface StoreManagerConfig {
     rpcCommunicator: RoutingRpcCommunicator
@@ -25,7 +32,7 @@ interface StoreManagerConfig {
     serviceId: ServiceID
     highestTtl: number
     redundancyFactor: number
-    getClosestNeighborsTo: (dataKey: DhtAddress, n?: number) => PeerDescriptor[]
+    getNeighbors: () => ReadonlyArray<PeerDescriptor>
     createRpcRemote: (contact: PeerDescriptor) => StoreRpcRemote
 }
 
@@ -42,9 +49,10 @@ export class StoreManager {
 
     private registerLocalRpcMethods() {
         const rpcLocal = new StoreRpcLocal({
+            localPeerDescriptor: this.config.localPeerDescriptor,
             localDataStore: this.config.localDataStore,
-            replicateDataToNeighbors: (incomingPeer: PeerDescriptor, dataEntry: DataEntry) => this.replicateDataToNeighbors(incomingPeer, dataEntry),
-            selfIsWithinRedundancyFactor: (key: DhtAddress): boolean => this.selfIsWithinRedundancyFactor(key)
+            replicateDataToContact: (dataEntry: DataEntry, contact: PeerDescriptor) => this.replicateDataToContact(dataEntry, contact),
+            getStorers: (dataKey: DhtAddress) => this.getStorers(dataKey)
         })
         this.config.rpcCommunicator.registerRpcMethod(StoreDataRequest, StoreDataResponse, 'storeData',
             (request: StoreDataRequest) => rpcLocal.storeData(request))
@@ -58,31 +66,19 @@ export class StoreManager {
         }
     }
 
-    private replicateAndUpdateStaleState(key: DhtAddress, newNode: PeerDescriptor): void {
-        const newNodeId = getNodeIdFromPeerDescriptor(newNode)
-        const closestToData = this.config.getClosestNeighborsTo(key, this.config.redundancyFactor)
-        const sortedList = new SortedContactList<Contact>({
-            referenceId: key, 
-            maxSize: this.config.redundancyFactor,
-            allowToContainReferenceId: true
-        })
-        sortedList.addContact(new Contact(this.config.localPeerDescriptor))
-        closestToData.forEach((neighbor) => {
-            if (newNodeId !== getNodeIdFromPeerDescriptor(neighbor)) {
-                sortedList.addContact(new Contact(neighbor))
-            }
-        })
-        const selfIsPrimaryStorer = (sortedList.getClosestContactId() === getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor))
-        if (selfIsPrimaryStorer) {
-            sortedList.addContact(new Contact(newNode))
-            if (sortedList.getContact(newNodeId) !== undefined) {
+    private replicateAndUpdateStaleState(dataKey: DhtAddress, newNode: PeerDescriptor): void {
+        const storers = this.getStorers(dataKey)
+        const storersBeforeContactAdded = storers.filter((p) => !areEqualPeerDescriptors(p, newNode))
+        const selfWasPrimaryStorer = areEqualPeerDescriptors(storersBeforeContactAdded[0], this.config.localPeerDescriptor)
+        if (selfWasPrimaryStorer) {
+            if (storers.some((p) => areEqualPeerDescriptors(p, newNode))) {
                 setImmediate(async () => {
-                    const dataEntries = Array.from(this.config.localDataStore.values(key))
+                    const dataEntries = Array.from(this.config.localDataStore.values(dataKey))
                     await Promise.all(dataEntries.map(async (dataEntry) => this.replicateDataToContact(dataEntry, newNode)))
                 })
             }
-        } else if (!this.selfIsWithinRedundancyFactor(key)) {
-            this.config.localDataStore.setAllEntriesAsStale(key)
+        } else if (!storers.some((p) => areEqualPeerDescriptors(p, this.config.localPeerDescriptor))) {
+            this.config.localDataStore.setAllEntriesAsStale(dataKey)
         }
     }
 
@@ -137,21 +133,15 @@ export class StoreManager {
         return successfulNodes
     }
 
-    private selfIsWithinRedundancyFactor(dataKey: DhtAddress): boolean {
-        const closestNeighbors = this.config.getClosestNeighborsTo(dataKey, this.config.redundancyFactor)
-        if (closestNeighbors.length < this.config.redundancyFactor) {
-            return true
-        } else {
-            const furthestCloseNeighbor = closestNeighbors[closestNeighbors.length - 1]
-            const dataKeyRaw = getRawFromDhtAddress(dataKey)
-            return getDistance(dataKeyRaw, this.config.localPeerDescriptor.nodeId) < getDistance(dataKeyRaw, furthestCloseNeighbor.nodeId)
-        }
-    }
-
     private async replicateDataToClosestNodes(): Promise<void> {
         const dataEntries = Array.from(this.config.localDataStore.values())
         await Promise.all(dataEntries.map(async (dataEntry) => {
-            const neighbors = this.config.getClosestNeighborsTo(getDhtAddressFromRaw(dataEntry.key), this.config.redundancyFactor)
+            const dataKey = getDhtAddressFromRaw(dataEntry.key)
+            const neighbors = getClosestNodes(
+                dataKey,
+                this.config.getNeighbors(),
+                { maxCount: this.config.redundancyFactor }
+            )
             await Promise.all(neighbors.map(async (neighbor) => {
                 const rpcRemote = this.config.createRpcRemote(neighbor)
                 try {
@@ -163,43 +153,15 @@ export class StoreManager {
         }))
     }
 
-    private replicateDataToNeighbors(incomingPeer: PeerDescriptor, dataEntry: DataEntry): void {
-        // sort own contact list according to data id
-        const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
-        const incomingNodeId = getNodeIdFromPeerDescriptor(incomingPeer)
-        const key = getDhtAddressFromRaw(dataEntry.key)
-        // TODO use config option or named constant?
-        const closestToData = this.config.getClosestNeighborsTo(key, 10)
-        const sortedList = new SortedContactList<Contact>({
-            referenceId: key, 
-            maxSize: this.config.redundancyFactor,
-            allowToContainReferenceId: true
-        })
-        sortedList.addContact(new Contact(this.config.localPeerDescriptor))
-        closestToData.forEach((neighbor) => {
-            sortedList.addContact(new Contact(neighbor))
-        })
-        const selfIsPrimaryStorer = (sortedList.getClosestContactId() === localNodeId)
-        const targetLimit = selfIsPrimaryStorer
-            // if we are the closest to the data, replicate to all storageRedundancyFactor nearest
-            ? undefined
-            // if we are not the closest node to the data, replicate only to the closest one to the data
-            : 1
-        const targets = sortedList.getClosestContacts(targetLimit)
-        targets.forEach((contact) => {
-            const contactNodeId = contact.getNodeId()
-            if ((incomingNodeId !== contactNodeId) && (localNodeId !== contactNodeId)) {
-                setImmediate(() => {
-                    executeSafePromise(async () => {
-                        await this.replicateDataToContact(dataEntry, contact.getPeerDescriptor())
-                        logger.trace('replicateDataToContact() returned', { 
-                            node: contactNodeId,
-                            replicateOnlyToClosest: !selfIsPrimaryStorer
-                        })
-                    })
-                })
+    private getStorers(dataKey: DhtAddress, excludedNode?: PeerDescriptor): PeerDescriptor[] {
+        return getClosestNodes(
+            dataKey,
+            [...this.config.getNeighbors(), this.config.localPeerDescriptor],
+            { 
+                maxCount: this.config.redundancyFactor,
+                excludedNodeIds: excludedNode !== undefined ? new Set([getNodeIdFromPeerDescriptor(excludedNode)]) : undefined
             }
-        })
+        )
     }
 
     async destroy(): Promise<void> {

package/src/dht/store/StoreRpcLocal.ts

@@ -1,5 +1,5 @@
 import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
-import { Logger } from '@streamr/utils'
+import { Logger, executeSafePromise } from '@streamr/utils'
 import { Empty } from '../../proto/google/protobuf/empty'
 import { Timestamp } from '../../proto/google/protobuf/timestamp'
 import {
@@ -11,12 +11,13 @@ import {
 import { IStoreRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
 import { DhtCallContext } from '../../rpc-protocol/DhtCallContext'
 import { LocalDataStore } from './LocalDataStore'
-import { DhtAddress, getDhtAddressFromRaw } from '../../identifiers'
+import { areEqualPeerDescriptors, DhtAddress, getDhtAddressFromRaw } from '../../identifiers'
 
 interface StoreRpcLocalConfig {
     localDataStore: LocalDataStore
-    replicateDataToNeighbors: (incomingPeer: PeerDescriptor, dataEntry: DataEntry) => void
-    selfIsWithinRedundancyFactor: (key: DhtAddress) => boolean
+    localPeerDescriptor: PeerDescriptor
+    replicateDataToContact: (dataEntry: DataEntry, contact: PeerDescriptor) => Promise<void>
+    getStorers: (key: DhtAddress) => ReadonlyArray<PeerDescriptor>
 }
 
 const logger = new Logger(module)
@@ -32,7 +33,7 @@ export class StoreRpcLocal implements IStoreRpc {
     async storeData(request: StoreDataRequest): Promise<StoreDataResponse> {
         logger.trace('storeData()')
         const key = getDhtAddressFromRaw(request.key)
-        const selfIsWithinRedundancyFactor = this.config.selfIsWithinRedundancyFactor(key)
+        const isLocalNodeStorer = this.isLocalNodeStorer(key)
         this.config.localDataStore.storeEntry({ 
             key: request.key,
             data: request.data,
@@ -40,10 +41,10 @@ export class StoreRpcLocal implements IStoreRpc {
             createdAt: request.createdAt,
             storedAt: Timestamp.now(),
             ttl: request.ttl,
-            stale: !selfIsWithinRedundancyFactor,
+            stale: !isLocalNodeStorer,
             deleted: false
         })
-        if (!selfIsWithinRedundancyFactor) {
+        if (!isLocalNodeStorer) {
             this.config.localDataStore.setAllEntriesAsStale(key)
         }
         return {}
@@ -54,13 +55,35 @@ export class StoreRpcLocal implements IStoreRpc {
         const dataEntry = request.entry!
         const wasStored = this.config.localDataStore.storeEntry(dataEntry)
         if (wasStored) {
-            this.config.replicateDataToNeighbors((context as DhtCallContext).incomingSourceDescriptor!, request.entry!)
+            this.replicateDataToNeighbors((context as DhtCallContext).incomingSourceDescriptor!, request.entry!)
         }
         const key = getDhtAddressFromRaw(dataEntry.key)
-        if (!this.config.selfIsWithinRedundancyFactor(key)) {
+        if (!this.isLocalNodeStorer(key)) {
             this.config.localDataStore.setAllEntriesAsStale(key)
         }
         logger.trace('server-side replicateData() at end')
         return {}
     }
+
+    private isLocalNodeStorer(dataKey: DhtAddress): boolean {
+        return this.config.getStorers(dataKey).some((p) => areEqualPeerDescriptors(p, this.config.localPeerDescriptor))    
+    }
+
+    private replicateDataToNeighbors(requestor: PeerDescriptor, dataEntry: DataEntry): void {
+        const dataKey = getDhtAddressFromRaw(dataEntry.key)
+        const storers = this.config.getStorers(dataKey)
+        const isLocalNodePrimaryStorer = areEqualPeerDescriptors(storers[0], this.config.localPeerDescriptor)
+        // If we are the closest to the data, get storageRedundancyFactor - 1 nearest node to the data, and
+        // replicate to all those node. Otherwise replicate only to the one closest one. And never replicate 
+        // to the requestor nor to itself.
+        const targets = (isLocalNodePrimaryStorer ? storers : [storers[0]]).filter(
+            (p) => !areEqualPeerDescriptors(p, requestor) && !areEqualPeerDescriptors(p, this.config.localPeerDescriptor)
+        )
+        targets.forEach((target) => {
+            setImmediate(() => {
+                executeSafePromise(() => this.config.replicateDataToContact(dataEntry, target))
+            })
+        })
+    }
+
 }

package/src/exports.ts

@@ -5,8 +5,9 @@ export { Simulator, LatencyType } from './connection/simulator/Simulator'
 export { SimulatorTransport } from './connection/simulator/SimulatorTransport'
 export { getRandomRegion, getRegionDelayMatrix } from './connection/simulator/pings'
 export { PeerDescriptor, Message, NodeType, DataEntry } from './proto/packages/dht/protos/DhtRpc'
-export { ITransport } from './transport/ITransport'
+export { ITransport, TransportEvents } from './transport/ITransport'
 export { ConnectionManager, ConnectionLocker, PortRange, TlsCertificate } from './connection/ConnectionManager'
+export { ConnectionsView } from './connection/ConnectionsView'
 export { LockID } from './connection/ConnectionLockStates'
 export { DefaultConnectorFacade } from './connection/ConnectorFacade'
 export { DhtRpcOptions } from './rpc-protocol/DhtRpcOptions'

package/src/helpers/version.ts

@@ -1,4 +1,4 @@
-export const LOCAL_PROTOCOL_VERSION = '1.0'
+export const LOCAL_PROTOCOL_VERSION = '1.1'
 
 /*
  * When two nodes negotiate whether they are compatible or not, it is up to the

package/src/proto/google/protobuf/any.ts

@@ -74,7 +74,7 @@ import { MessageType } from "@protobuf-ts/runtime";
  *       foo = any.unpack(Foo.class);
  *     }
  *
- *  Example 3: Pack and unpack a message in Python.
+ * Example 3: Pack and unpack a message in Python.
  *
  *     foo = Foo(...)
  *     any = Any()
@@ -84,13 +84,16 @@ import { MessageType } from "@protobuf-ts/runtime";
  *       any.Unpack(foo)
  *       ...
  *
- *  Example 4: Pack and unpack a message in Go
+ * Example 4: Pack and unpack a message in Go
  *
  *      foo := &pb.Foo{...}
- *      any, err := ptypes.MarshalAny(foo)
+ *      any, err := anypb.New(foo)
+ *      if err != nil {
+ *        ...
+ *      }
  *      ...
  *      foo := &pb.Foo{}
- *      if err := ptypes.UnmarshalAny(any, foo); err != nil {
+ *      if err := any.UnmarshalTo(foo); err != nil {
  *        ...
  *      }
  *
@@ -102,7 +105,7 @@ import { MessageType } from "@protobuf-ts/runtime";
  *
  *
  * JSON
- * ====
+ *
  * The JSON representation of an `Any` value uses the regular
  * representation of the deserialized, embedded message, with an
  * additional field `@type` which contains the type URL. Example:

package/src/proto/google/protobuf/empty.ts

@@ -49,7 +49,6 @@ import { MessageType } from "@protobuf-ts/runtime";
  *       rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty);
  *     }
  *
- * The JSON representation for `Empty` is empty JSON object `{}`.
  *
  * @generated from protobuf message google.protobuf.Empty
  */

package/src/proto/google/protobuf/timestamp.ts

@@ -98,7 +98,16 @@ import { MessageType } from "@protobuf-ts/runtime";
  *         .setNanos((int) ((millis % 1000) * 1000000)).build();
  *
  *
- * Example 5: Compute Timestamp from current time in Python.
+ * Example 5: Compute Timestamp from Java `Instant.now()`.
+ *
+ *     Instant now = Instant.now();
+ *
+ *     Timestamp timestamp =
+ *         Timestamp.newBuilder().setSeconds(now.getEpochSecond())
+ *             .setNanos(now.getNano()).build();
+ *
+ *
+ * Example 6: Compute Timestamp from current time in Python.
  *
  *     timestamp = Timestamp()
  *     timestamp.GetCurrentTime()

package/src/proto/packages/dht/protos/DhtRpc.ts

@@ -351,9 +351,9 @@ export interface ConnectivityRequest {
      */
     host?: string;
     /**
-     * @generated from protobuf field: bool selfSigned = 4;
+     * @generated from protobuf field: bool allowSelfSignedCertificate = 4;
      */
-    selfSigned: boolean;
+    allowSelfSignedCertificate: boolean;
 }
 /**
  * @generated from protobuf message dht.ConnectivityResponse
@@ -379,6 +379,14 @@ export interface ConnectivityResponse {
      * @generated from protobuf field: string version = 5;
      */
     version: string;
+    /**
+     * @generated from protobuf field: optional double latitude = 6;
+     */
+    latitude?: number;
+    /**
+     * @generated from protobuf field: optional double longitude = 7;
+     */
+    longitude?: number;
 }
 /**
  * @generated from protobuf message dht.HandshakeRequest
@@ -962,7 +970,7 @@ class ConnectivityRequest$Type extends MessageType<ConnectivityRequest> {
             { no: 1, name: "port", kind: "scalar", T: 13 /*ScalarType.UINT32*/ },
             { no: 2, name: "tls", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
             { no: 3, name: "host", kind: "scalar", opt: true, T: 9 /*ScalarType.STRING*/ },
-            { no: 4, name: "selfSigned", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
+            { no: 4, name: "allowSelfSignedCertificate", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
         ]);
     }
 }
@@ -978,7 +986,9 @@ class ConnectivityResponse$Type extends MessageType<ConnectivityResponse> {
             { no: 2, name: "natType", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
             { no: 3, name: "websocket", kind: "message", T: () => ConnectivityMethod },
             { no: 4, name: "ipAddress", kind: "scalar", T: 13 /*ScalarType.UINT32*/ },
-            { no: 5, name: "version", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
+            { no: 5, name: "version", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
+            { no: 6, name: "latitude", kind: "scalar", opt: true, T: 1 /*ScalarType.DOUBLE*/ },
+            { no: 7, name: "longitude", kind: "scalar", opt: true, T: 1 /*ScalarType.DOUBLE*/ }
         ]);
     }
 }

package/src/transport/ITransport.ts

@@ -1,4 +1,3 @@
-import { DhtAddress } from '../identifiers'
 import { Message, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
 
 export interface TransportEvents {
@@ -33,8 +32,5 @@ export interface ITransport {
 
     send(msg: Message, opts?: SendOptions): Promise<void>
     getLocalPeerDescriptor(): PeerDescriptor
-    getConnections(): PeerDescriptor[]
-    getConnectionCount(): number
-    hasConnection(nodeId: DhtAddress): boolean
     stop(): void | Promise<void>
 }

package/test/benchmark/Find.test.ts

@@ -56,7 +56,7 @@ describe('Find correctness', () => {
         logger.info('waiting over')
 
         nodes.forEach((node) => logger.info(getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + ': connections:' +
-            node.getConnectionCount() + ', kbucket: ' + node.getNeighborCount()
+            node.getConnectionsView().getConnectionCount() + ', kbucket: ' + node.getNeighborCount()
             + ', localLocked: ' + node.getLocalLockedConnectionCount()
             + ', remoteLocked: ' + node.getRemoteLockedConnectionCount()
             + ', weakLocked: ' + node.getWeakLockedConnectionCount()))

package/test/benchmark/WebsocketServerMemoryLeak.test.ts

@@ -31,7 +31,7 @@ describe('WebsocketServermemoryLeak', () => {
                 console.log('clientWebsocket connected ' + i)
             })
             
-            clientWebsocket.connect(`ws://127.0.0.1:${port}`)
+            clientWebsocket.connect(`ws://127.0.0.1:${port}`, false)
             i++
             await wait(3000)
         }

package/test/end-to-end/GeoIpLayer0.test.ts

@@ -0,0 +1,55 @@
+import { WebsocketServerConnection } from '../../src/connection/websocket/WebsocketServerConnection'
+import { DhtNode } from '../../src/dht/DhtNode'
+import fs from 'fs'
+
+const WEBSOCKET_PORT_RANGE = { min: 10012, max: 10015 }
+
+// '51.120.98.194' is the IP address of norway.no in OSL = 7900
+const testIp = '51.120.98.194'
+const testRegion = 7900
+const dbPath = '/tmp/geoipdatabasesl0'
+
+describe('Layer0', () => {
+
+    let epDhtNode: DhtNode
+    let node1: DhtNode
+    let mock: jest.SpyInstance<string, [], any> 
+    
+    beforeEach(async () => {
+
+        epDhtNode = new DhtNode({
+            websocketHost: '127.0.0.1', websocketPortRange: { min: 10011, max: 10011 }, websocketServerEnableTls: false,
+            geoIpDatabaseFolder: dbPath
+        })
+        await epDhtNode.start()
+        await epDhtNode.joinDht([epDhtNode.getLocalPeerDescriptor()])
+
+        node1 = new DhtNode({
+            websocketPortRange: WEBSOCKET_PORT_RANGE,
+            websocketHost: '127.0.0.1',
+            entryPoints: [epDhtNode.getLocalPeerDescriptor()],
+            websocketServerEnableTls: false
+        })
+
+        mock = jest.spyOn(WebsocketServerConnection.prototype, 'getRemoteIpAddress').mockImplementation(() => testIp)
+
+    }, 10000)
+
+    afterEach(async () => {
+        await Promise.all([
+            epDhtNode.stop(),
+            node1.stop()
+        ])
+        fs.unlinkSync(dbPath + '/GeoLite2-City.mmdb')
+        fs.rmSync(dbPath, { recursive: true })
+        mock.mockRestore()
+    })
+
+    it('Gets the correct region number by IP address', async () => {
+        await node1.start()
+        await node1.joinDht([epDhtNode.getLocalPeerDescriptor()])
+        
+        expect(node1.getLocalPeerDescriptor().region).toBe(testRegion)
+
+    }, 10000)
+})

package/test/end-to-end/Layer0-Layer1.test.ts

@@ -39,11 +39,11 @@ describe('Layer0-Layer1', () => {
         await node1.start()
         await node2.start()
 
-        stream1Node1 = new DhtNode({ transport: epDhtNode, serviceId: STREAM_ID1 })
-        stream1Node2 = new DhtNode({ transport: node1, serviceId: STREAM_ID1 })
+        stream1Node1 = new DhtNode({ transport: epDhtNode, connectionsView: epDhtNode.getConnectionsView(), serviceId: STREAM_ID1 })
+        stream1Node2 = new DhtNode({ transport: node1, connectionsView: node1.getConnectionsView(), serviceId: STREAM_ID1 })
 
-        stream2Node1 = new DhtNode({ transport: epDhtNode, serviceId: STREAM_ID2 })
-        stream2Node2 = new DhtNode({ transport: node2, serviceId: STREAM_ID2 })
+        stream2Node1 = new DhtNode({ transport: epDhtNode, connectionsView: epDhtNode.getConnectionsView(), serviceId: STREAM_ID2 })
+        stream2Node2 = new DhtNode({ transport: node2, connectionsView: node2.getConnectionsView(), serviceId: STREAM_ID2 })
 
         await Promise.all([
             stream1Node1.start(),