@streamr/dht 100.0.0-testnet-one.4 → 100.0.0-testnet-two.0

package/src/dht/DhtNode.ts

@@ -17,11 +17,11 @@ import {
     ExternalFindDataResponse,
     ExternalStoreDataRequest,
     ExternalStoreDataResponse,
-    FindAction,
+    RecursiveOperation,
 } from '../proto/packages/dht/protos/DhtRpc'
 import { ITransport, TransportEvents } from '../transport/ITransport'
 import { ConnectionManager, PortRange, TlsCertificate } from '../connection/ConnectionManager'
-import { DhtNodeRpcClient, ExternalApiRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
+import { ExternalApiRpcClient, StoreRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
 import {
     Logger,
     MetricsContext,
@@ -29,16 +29,13 @@ import {
     merge,
     waitForCondition
 } from '@streamr/utils'
-import { toProtoRpcClient } from '@streamr/proto-rpc'
 import { Any } from '../proto/google/protobuf/any'
 import {
-    areEqualPeerDescriptors,
-    getNodeIdFromPeerDescriptor,
-    peerIdFromPeerDescriptor
+    getNodeIdFromPeerDescriptor
 } from '../helpers/peerIdFromPeerDescriptor'
 import { Router } from './routing/Router'
-import { Finder, FindResult } from './find/Finder'
-import { StoreRpcLocal } from './store/StoreRpcLocal'
+import { RecursiveOperationManager, RecursiveOperationResult } from './recursive-operation/RecursiveOperationManager'
+import { StoreManager } from './store/StoreManager'
 import { PeerDiscovery } from './discovery/PeerDiscovery'
 import { LocalDataStore } from './store/LocalDataStore'
 import { IceServer } from '../connection/webrtc/WebrtcConnector'
@@ -51,18 +48,20 @@ import { MarkRequired } from 'ts-essentials'
 import { DhtNodeRpcLocal } from './DhtNodeRpcLocal'
 import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
 import { ExternalApiRpcLocal } from './ExternalApiRpcLocal'
-import { PeerManager, getDistance } from './PeerManager'
+import { PeerManager } from './PeerManager'
+import { ServiceID } from '../types/ServiceID'
+import { NodeID, getNodeIdFromBinary } from '../helpers/nodeId'
+import { StoreRpcRemote } from './store/StoreRpcRemote'
 
 export interface DhtNodeEvents {
     newContact: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
     contactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
-    joinCompleted: () => void
     newRandomContact: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
     randomContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
 }
 
 export interface DhtNodeOptions {
-    serviceId?: string
+    serviceId?: ServiceID
     joinParallelism?: number
     maxNeighborListSize?: number
     numberOfNodesPerKBucket?: number
@@ -140,16 +139,14 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
     private transport?: ITransport
     private localPeerDescriptor?: PeerDescriptor
     public router?: Router
-    private storeRpcLocal?: StoreRpcLocal
-    private localDataStore = new LocalDataStore()
-    private finder?: Finder
+    private storeManager?: StoreManager
+    private localDataStore: LocalDataStore
+    private recursiveOperationManager?: RecursiveOperationManager
     private peerDiscovery?: PeerDiscovery
     private peerManager?: PeerManager
-
     public connectionManager?: ConnectionManager
     private started = false
     private abortController = new AbortController()
-    private entryPointDisconnectTimeout?: NodeJS.Timeout
 
     constructor(conf: DhtNodeOptions) {
         super()
@@ -169,6 +166,7 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             metricsContext: new MetricsContext(),
             peerId: new UUID().toHex()
         }, conf)
+        this.localDataStore = new LocalDataStore(this.config.storeMaxTtl) 
         this.send = this.send.bind(this)
     }
 
@@ -258,10 +256,9 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             connections: this.peerManager!.connections,
             localPeerDescriptor: this.localPeerDescriptor!,
             addContact: (contact: PeerDescriptor, setActive?: boolean) => this.peerManager!.handleNewPeers([contact], setActive),
-            serviceId: this.config.serviceId,
             connectionManager: this.connectionManager
         })
-        this.finder = new Finder({
+        this.recursiveOperationManager = new RecursiveOperationManager({
             rpcCommunicator: this.rpcCommunicator,
             router: this.router,
             sessionTransport: this,
@@ -269,36 +266,41 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             localPeerDescriptor: this.localPeerDescriptor!,
             serviceId: this.config.serviceId,
             addContact: (contact: PeerDescriptor) => this.peerManager!.handleNewPeers([contact]),
-            isPeerCloserToIdThanSelf: this.isPeerCloserToIdThanSelf.bind(this),
             localDataStore: this.localDataStore
         })
-        this.storeRpcLocal = new StoreRpcLocal({
+        this.storeManager = new StoreManager({
             rpcCommunicator: this.rpcCommunicator,
-            finder: this.finder,
+            recursiveOperationManager: this.recursiveOperationManager,
             localPeerDescriptor: this.localPeerDescriptor!,
             serviceId: this.config.serviceId,
             highestTtl: this.config.storeHighestTtl,
-            maxTtl: this.config.storeMaxTtl,
             redundancyFactor: this.config.storageRedundancyFactor,
             localDataStore: this.localDataStore,
-            dhtNodeEmitter: this,
-            getNodesClosestToIdFromBucket: (id: Uint8Array, n?: number) => {
-                return this.peerManager!.getClosestNeighborsTo(id, n)
+            getClosestNeighborsTo: (id: Uint8Array, n?: number) => {
+                return this.peerManager!.getClosestNeighborsTo(getNodeIdFromBinary(id), n).map((n) => n.getPeerDescriptor())
             },
-            rpcRequestTimeout: this.config.rpcRequestTimeout
+            createRpcRemote: (contact: PeerDescriptor) => {
+                return new StoreRpcRemote(
+                    this.localPeerDescriptor!,
+                    contact,
+                    this.config.serviceId,
+                    this.rpcCommunicator!,
+                    StoreRpcClient,
+                    this.config.rpcRequestTimeout
+                )
+            }
+        })
+        this.on('newContact', (peerDescriptor: PeerDescriptor) => {
+            this.storeManager!.onNewContact(peerDescriptor)
         })
         this.bindRpcLocalMethods()
-        if ((this.connectionManager !== undefined) && (this.config.entryPoints !== undefined) && this.config.entryPoints.length > 0
-            && !areEqualPeerDescriptors(this.config.entryPoints[0], this.localPeerDescriptor!)) {
-            this.connectToEntryPoint(this.config.entryPoints[0])
-        }
     }
 
     private initPeerManager() {
         this.peerManager = new PeerManager({
             numberOfNodesPerKBucket: this.config.numberOfNodesPerKBucket,
             maxContactListSize: this.config.maxNeighborListSize,
-            ownPeerId: this.getNodeId(),
+            localNodeId: this.getNodeId(),
             connectionManager: this.connectionManager!,
             peerDiscoveryQueryBatchSize: this.config.peerDiscoveryQueryBatchSize,
             isLayer0: (this.connectionManager !== undefined),
@@ -347,10 +349,9 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             return
         }
         const dhtNodeRpcLocal = new DhtNodeRpcLocal({
-            serviceId: this.config.serviceId,
             peerDiscoveryQueryBatchSize: this.config.peerDiscoveryQueryBatchSize,
             getClosestPeersTo: (kademliaId: Uint8Array, limit: number) => {
-                return this.peerManager!.getClosestNeighborsTo(kademliaId, limit)
+                return this.peerManager!.getClosestNeighborsTo(getNodeIdFromBinary(kademliaId), limit)
                     .map((dhtPeer: DhtNodeRpcRemote) => dhtPeer.getPeerDescriptor())
             },
             addNewContact: (contact: PeerDescriptor) => this.peerManager!.handleNewPeers([contact]),
@@ -363,33 +364,27 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
         this.rpcCommunicator!.registerRpcNotification(LeaveNotice, 'leaveNotice',
             (req: LeaveNotice, context) => dhtNodeRpcLocal.leaveNotice(req, context))
         const externalApiRpcLocal = new ExternalApiRpcLocal({
-            startFind: (key: Uint8Array, action: FindAction, excludedPeer: PeerDescriptor) => {
-                return this.startFind(key, action, excludedPeer)
+            executeRecursiveOperation: (key: Uint8Array, operation: RecursiveOperation, excludedPeer: PeerDescriptor) => {
+                return this.executeRecursiveOperation(key, operation, excludedPeer)
             },
-            storeDataToDht: (key: Uint8Array, data: Any, creator?: PeerDescriptor) => this.storeDataToDht(key, data, creator)
+            storeDataToDht: (key: Uint8Array, data: Any, creator?: NodeID) => this.storeDataToDht(key, data, creator)
         })
         this.rpcCommunicator!.registerRpcMethod(
             ExternalFindDataRequest,
             ExternalFindDataResponse,
             'externalFindData',
             (req: ExternalFindDataRequest, context: ServerCallContext) => externalApiRpcLocal.externalFindData(req, context),
-            { timeout: 10000 }
+            { timeout: 10000 }  // TODO use config option or named constant?
         )
         this.rpcCommunicator!.registerRpcMethod(
             ExternalStoreDataRequest,
             ExternalStoreDataResponse,
             'externalStoreData',
             (req: ExternalStoreDataRequest, context: ServerCallContext) => externalApiRpcLocal.externalStoreData(req, context),
-            { timeout: 10000 }
+            { timeout: 10000 }  // TODO use config option or named constant?
         )
     }
 
-    private isPeerCloserToIdThanSelf(peer1: PeerDescriptor, compareToId: PeerID): boolean {
-        const distance1 = getDistance(peer1.nodeId, compareToId.value)
-        const distance2 = getDistance(this.localPeerDescriptor!.nodeId, compareToId.value)
-        return distance1 < distance2
-    }
-
     private handleMessage(message: Message): void {
         if (message.serviceId === this.config.serviceId) {
             logger.trace('callig this.handleMessageFromPeer ' + getNodeIdFromPeerDescriptor(message.sourceDescriptor!)
@@ -412,24 +407,20 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
     }
 
     public getClosestContacts(limit?: number): PeerDescriptor[] {
-        return this.peerManager!.getClosestContactsTo(this.localPeerDescriptor!.nodeId, limit).map((peer) => peer.getPeerDescriptor())
+        return this.peerManager!.getClosestContactsTo(
+            getNodeIdFromPeerDescriptor(this.localPeerDescriptor!),
+            limit).map((peer) => peer.getPeerDescriptor()
+        )
     }
     
-    public getNodeId(): PeerID {
-        return peerIdFromPeerDescriptor(this.localPeerDescriptor!)
+    public getNodeId(): NodeID {
+        return getNodeIdFromPeerDescriptor(this.localPeerDescriptor!)
     }
 
     public getNumberOfNeighbors(): number {
         return this.peerManager!.getNumberOfNeighbors()
     }
 
-    private connectToEntryPoint(entryPoint: PeerDescriptor): void {
-        this.connectionManager!.lockConnection(entryPoint, 'temporary-layer0-connection')
-        this.entryPointDisconnectTimeout = setTimeout(() => {
-            this.connectionManager!.unlockConnection(entryPoint, 'temporary-layer0-connection')
-        }, 10 * 1000)
-    }
-
     public removeContact(contact: PeerDescriptor): void {
         if (!this.started) {  // the stopped state is checked in PeerManager
             return
@@ -449,18 +440,26 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
         if (!this.started) {
             throw new Error('Cannot join DHT before calling start() on DhtNode')
         }
-        await this.peerDiscovery!.joinDht(entryPointDescriptors, doAdditionalRandomPeerDiscovery, retry)
+        await Promise.all(entryPointDescriptors.map((entryPoint) =>
+            this.peerDiscovery!.joinDht(entryPoint, doAdditionalRandomPeerDiscovery, retry)
+        ))
     }
 
-    public async startFind(key: Uint8Array, action?: FindAction, excludedPeer?: PeerDescriptor): Promise<FindResult> {
-        return this.finder!.startFind(key, action, excludedPeer)
+    // TODO make this private and unify the public API of find/fetch/store/delete methods
+    // (we already have storeDataToDht etc. here)
+    public async executeRecursiveOperation(
+        key: Uint8Array,
+        operation: RecursiveOperation,
+        excludedPeer?: PeerDescriptor
+    ): Promise<RecursiveOperationResult> {
+        return this.recursiveOperationManager!.execute(key, operation, excludedPeer)
     }
 
-    public async storeDataToDht(key: Uint8Array, data: Any, creator?: PeerDescriptor): Promise<PeerDescriptor[]> {
+    public async storeDataToDht(key: Uint8Array, data: Any, creator?: NodeID): Promise<PeerDescriptor[]> {
         if (this.peerDiscovery!.isJoinOngoing() && this.config.entryPoints && this.config.entryPoints.length > 0) {
             return this.storeDataViaPeer(key, data, sample(this.config.entryPoints)!)
         }
-        return this.storeRpcLocal!.storeDataToDht(key, data, creator ?? this.localPeerDescriptor!)
+        return this.storeManager!.storeDataToDht(key, data, creator ?? getNodeIdFromPeerDescriptor(this.localPeerDescriptor!))
     }
 
     public async storeDataViaPeer(key: Uint8Array, data: Any, peer: PeerDescriptor): Promise<PeerDescriptor[]> {
@@ -468,7 +467,8 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             this.localPeerDescriptor!,
             peer,
             this.config.serviceId,
-            toProtoRpcClient(new ExternalApiRpcClient(this.rpcCommunicator!.getRpcClientTransport()))
+            this.rpcCommunicator!,
+            ExternalApiRpcClient
         )
         return await rpcRemote.storeData(key, data)
     }
@@ -477,13 +477,13 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
         if (this.peerDiscovery!.isJoinOngoing() && this.config.entryPoints && this.config.entryPoints.length > 0) {
             return this.findDataViaPeer(key, sample(this.config.entryPoints)!)
         }
-        const result = await this.finder!.startFind(key, FindAction.FETCH_DATA)
+        const result = await this.recursiveOperationManager!.execute(key, RecursiveOperation.FETCH_DATA)
         return result.dataEntries ?? []  // TODO is this fallback needed?
     }
 
     public async deleteDataFromDht(key: Uint8Array, waitForCompletion: boolean): Promise<void> {
         if (!this.abortController.signal.aborted) {
-            await this.finder!.startFind(key, FindAction.DELETE_DATA, undefined, waitForCompletion)
+            await this.recursiveOperationManager!.execute(key, RecursiveOperation.DELETE_DATA, undefined, waitForCompletion)
         }
     }
 
@@ -492,7 +492,8 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             this.localPeerDescriptor!,
             peer,
             this.config.serviceId,
-            toProtoRpcClient(new ExternalApiRpcClient(this.rpcCommunicator!.getRpcClientTransport()))
+            this.rpcCommunicator!,
+            ExternalApiRpcClient
         )
         return await rpcRemote.externalFindData(key)
     }
@@ -550,15 +551,12 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
         }
         logger.trace('stop()')
         this.abortController.abort()
-        await this.storeRpcLocal!.destroy()
-        if (this.entryPointDisconnectTimeout) {
-            clearTimeout(this.entryPointDisconnectTimeout)
-        }
+        await this.storeManager!.destroy()
         this.localDataStore.clear()
         this.peerManager?.stop()
         this.rpcCommunicator!.stop()
         this.router!.stop()
-        this.finder!.stop()
+        this.recursiveOperationManager!.stop()
         this.peerDiscovery!.stop()
         if (this.config.transport === undefined) {
             // if the transport was not given in config, the instance was created in start() and
@@ -574,8 +572,8 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
         return new DhtNodeRpcRemote(
             this.localPeerDescriptor!,
             peerDescriptor,
-            toProtoRpcClient(new DhtNodeRpcClient(this.rpcCommunicator!.getRpcClientTransport())),
             this.config.serviceId,
+            this.rpcCommunicator!,
             this.config.rpcRequestTimeout
         )
     }

package/src/dht/DhtNodeRpcLocal.ts

@@ -14,7 +14,6 @@ import { IDhtNodeRpc } from '../proto/packages/dht/protos/DhtRpc.server'
 import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
 
 interface DhtNodeRpcLocalConfig {
-    serviceId: string
     peerDiscoveryQueryBatchSize: number
     getClosestPeersTo: (nodeId: Uint8Array, limit: number) => PeerDescriptor[]
     addNewContact: (contact: PeerDescriptor) => void

package/src/dht/DhtNodeRpcRemote.ts

@@ -1,17 +1,17 @@
-import { IDhtNodeRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
+import { RpcCommunicator } from '@streamr/proto-rpc'
+import { Logger } from '@streamr/utils'
+import { v4 } from 'uuid'
+import { NodeID } from '../helpers/nodeId'
+import { getNodeIdFromPeerDescriptor } from '../helpers/peerIdFromPeerDescriptor'
 import {
     ClosestPeersRequest,
     LeaveNotice,
     PeerDescriptor,
     PingRequest
 } from '../proto/packages/dht/protos/DhtRpc'
-import { v4 } from 'uuid'
-import { Logger } from '@streamr/utils'
-import { ProtoRpcClient } from '@streamr/proto-rpc'
-import { RpcRemote } from './contact/RpcRemote'
-import { PeerID } from '../helpers/PeerID'
-import { getNodeIdFromPeerDescriptor, peerIdFromPeerDescriptor } from '../helpers/peerIdFromPeerDescriptor'
+import { DhtNodeRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
 import { ServiceID } from '../types/ServiceID'
+import { RpcRemote } from './contact/RpcRemote'
 
 const logger = new Logger(module)
 
@@ -21,7 +21,7 @@ export interface KBucketContact {
     vectorClock: number
 }
 
-export class DhtNodeRpcRemote extends RpcRemote<IDhtNodeRpcClient> implements KBucketContact {
+export class DhtNodeRpcRemote extends RpcRemote<DhtNodeRpcClient> implements KBucketContact {
 
     private static counter = 0
     public vectorClock: number
@@ -30,12 +30,12 @@ export class DhtNodeRpcRemote extends RpcRemote<IDhtNodeRpcClient> implements KB
     constructor(
         localPeerDescriptor: PeerDescriptor,
         peerDescriptor: PeerDescriptor,
-        client: ProtoRpcClient<IDhtNodeRpcClient>,
         serviceId: ServiceID,
+        rpcCommunicator: RpcCommunicator,
         rpcRequestTimeout?: number
     ) {
-        super(localPeerDescriptor, peerDescriptor, serviceId, client, rpcRequestTimeout)
-        this.id = this.getPeerId().value
+        super(localPeerDescriptor, peerDescriptor, serviceId, rpcCommunicator, DhtNodeRpcClient, rpcRequestTimeout)
+        this.id = this.getPeerDescriptor().nodeId
         this.vectorClock = DhtNodeRpcRemote.counter++
     }
 
@@ -84,7 +84,7 @@ export class DhtNodeRpcRemote extends RpcRemote<IDhtNodeRpcClient> implements KB
         })
     }
 
-    getPeerId(): PeerID {
-        return peerIdFromPeerDescriptor(this.getPeerDescriptor())
+    getNodeId(): NodeID {
+        return getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
     }
 }

package/src/dht/ExternalApiRpcLocal.ts

@@ -4,17 +4,27 @@ import {
     ExternalFindDataResponse,
     ExternalStoreDataRequest,
     ExternalStoreDataResponse,
-    FindAction,
+    RecursiveOperation,
     PeerDescriptor
 } from '../proto/packages/dht/protos/DhtRpc'
 import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
 import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
-import { FindResult } from './find/Finder'
+import { RecursiveOperationResult } from './recursive-operation/RecursiveOperationManager'
 import { Any } from '../proto/google/protobuf/any'
+import { NodeID } from '../helpers/nodeId'
+import { getNodeIdFromPeerDescriptor } from '../helpers/peerIdFromPeerDescriptor'
 
 interface ExternalApiRpcLocalConfig {
-    startFind: (idToFind: Uint8Array, action: FindAction, excludedPeer: PeerDescriptor) => Promise<FindResult>
-    storeDataToDht: (key: Uint8Array, data: Any, creator: PeerDescriptor) => Promise<PeerDescriptor[]>
+    executeRecursiveOperation: (
+        targetId: Uint8Array,
+        operation: RecursiveOperation,
+        excludedPeer: PeerDescriptor
+    ) => Promise<RecursiveOperationResult>
+    storeDataToDht: (
+        key: Uint8Array,
+        data: Any,
+        creator: NodeID
+    ) => Promise<PeerDescriptor[]>
 }
 
 export class ExternalApiRpcLocal implements IExternalApiRpc {
@@ -27,13 +37,13 @@ export class ExternalApiRpcLocal implements IExternalApiRpc {
 
     async externalFindData(findDataRequest: ExternalFindDataRequest, context: ServerCallContext): Promise<ExternalFindDataResponse> {
         const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
-        const result = await this.config.startFind(findDataRequest.key, FindAction.FETCH_DATA, senderPeerDescriptor)
+        const result = await this.config.executeRecursiveOperation(findDataRequest.key, RecursiveOperation.FETCH_DATA, senderPeerDescriptor)
         return ExternalFindDataResponse.create({ entries: result.dataEntries ?? [] })
     }
 
     async externalStoreData(request: ExternalStoreDataRequest, context: ServerCallContext): Promise<ExternalStoreDataResponse> {
         const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
-        const result = await this.config.storeDataToDht(request.key, request.data!, senderPeerDescriptor)
+        const result = await this.config.storeDataToDht(request.key, request.data!, getNodeIdFromPeerDescriptor(senderPeerDescriptor))
         return ExternalStoreDataResponse.create({
             storers: result
         })

package/src/dht/ExternalApiRpcRemote.ts

@@ -1,15 +1,16 @@
 import { Any } from '../proto/google/protobuf/any'
-import { DataEntry, ExternalStoreDataRequest, ExternalFindDataRequest, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
-import { IExternalApiRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
+import { DataEntry, ExternalFindDataRequest, ExternalStoreDataRequest, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
+import { ExternalApiRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
 import { RpcRemote } from './contact/RpcRemote'
 
-export class ExternalApiRpcRemote extends RpcRemote<IExternalApiRpcClient> {
+export class ExternalApiRpcRemote extends RpcRemote<ExternalApiRpcClient> {
 
     async externalFindData(key: Uint8Array): Promise<DataEntry[]> {
         const request: ExternalFindDataRequest = {
             key
         }
         const options = this.formDhtRpcOptions({
+            // TODO use config option or named constant?
             timeout: 10000
         })
         try {
@@ -26,6 +27,7 @@ export class ExternalApiRpcRemote extends RpcRemote<IExternalApiRpcClient> {
             data
         }
         const options = this.formDhtRpcOptions({
+            // TODO use config option or named constant?
             timeout: 10000
         })
         try {