@streamr/dht 100.0.0-pretestnet.4 → 100.0.0-rc.0

package/src/dht/store/StoreManager.ts

@@ -0,0 +1,209 @@
+import {
+    DataEntry, ReplicateDataRequest, PeerDescriptor,
+    StoreDataRequest, StoreDataResponse, RecursiveOperation
+} from '../../proto/packages/dht/protos/DhtRpc'
+import { Any } from '../../proto/google/protobuf/any'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
+import { RecursiveOperationManager } from '../recursive-operation/RecursiveOperationManager'
+import { Logger, executeSafePromise } from '@streamr/utils'
+import { LocalDataStore } from './LocalDataStore'
+import { StoreRpcRemote } from './StoreRpcRemote'
+import { Timestamp } from '../../proto/google/protobuf/timestamp'
+import { SortedContactList } from '../contact/SortedContactList'
+import { Contact } from '../contact/Contact'
+import { ServiceID } from '../../types/ServiceID'
+import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
+import { StoreRpcLocal } from './StoreRpcLocal'
+import { getDistance } from '../PeerManager'
+
+interface StoreManagerConfig {
+    rpcCommunicator: RoutingRpcCommunicator
+    recursiveOperationManager: RecursiveOperationManager
+    localPeerDescriptor: PeerDescriptor
+    localDataStore: LocalDataStore
+    serviceId: ServiceID
+    highestTtl: number
+    redundancyFactor: number
+    getClosestNeighborsTo: (dataKey: DhtAddress, n?: number) => PeerDescriptor[]
+    createRpcRemote: (contact: PeerDescriptor) => StoreRpcRemote
+}
+
+const logger = new Logger(module)
+
+export class StoreManager {
+
+    private readonly config: StoreManagerConfig
+
+    constructor(config: StoreManagerConfig) {
+        this.config = config
+        this.registerLocalRpcMethods()
+    }
+
+    private registerLocalRpcMethods() {
+        const rpcLocal = new StoreRpcLocal({
+            localDataStore: this.config.localDataStore,
+            replicateDataToNeighbors: (incomingPeer: PeerDescriptor, dataEntry: DataEntry) => this.replicateDataToNeighbors(incomingPeer, dataEntry),
+            selfIsWithinRedundancyFactor: (key: DhtAddress): boolean => this.selfIsWithinRedundancyFactor(key)
+        })
+        this.config.rpcCommunicator.registerRpcMethod(StoreDataRequest, StoreDataResponse, 'storeData',
+            (request: StoreDataRequest) => rpcLocal.storeData(request))
+        this.config.rpcCommunicator.registerRpcNotification(ReplicateDataRequest, 'replicateData',
+            (request: ReplicateDataRequest, context: ServerCallContext) => rpcLocal.replicateData(request, context))
+    }
+
+    onContactAdded(peerDescriptor: PeerDescriptor): void {
+        for (const key of this.config.localDataStore.keys()) {
+            this.replicateAndUpdateStaleState(key, peerDescriptor)
+        }
+    }
+
+    private replicateAndUpdateStaleState(key: DhtAddress, newNode: PeerDescriptor): void {
+        const newNodeId = getNodeIdFromPeerDescriptor(newNode)
+        const closestToData = this.config.getClosestNeighborsTo(key, this.config.redundancyFactor)
+        const sortedList = new SortedContactList<Contact>({
+            referenceId: key, 
+            maxSize: this.config.redundancyFactor,
+            allowToContainReferenceId: true,
+            emitEvents: false
+        })
+        sortedList.addContact(new Contact(this.config.localPeerDescriptor))
+        closestToData.forEach((neighbor) => {
+            if (newNodeId !== getNodeIdFromPeerDescriptor(neighbor)) {
+                sortedList.addContact(new Contact(neighbor))
+            }
+        })
+        const selfIsPrimaryStorer = (sortedList.getClosestContactId() === getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor))
+        if (selfIsPrimaryStorer) {
+            sortedList.addContact(new Contact(newNode))
+            if (sortedList.getContact(newNodeId) !== undefined) {
+                setImmediate(async () => {
+                    const dataEntries = Array.from(this.config.localDataStore.values(key))
+                    await Promise.all(dataEntries.map(async (dataEntry) => this.replicateDataToContact(dataEntry, newNode)))
+                })
+            }
+        } else if (!this.selfIsWithinRedundancyFactor(key)) {
+            this.config.localDataStore.setAllEntriesAsStale(key)
+        }
+    }
+
+    private async replicateDataToContact(dataEntry: DataEntry, contact: PeerDescriptor): Promise<void> {
+        const rpcRemote = this.config.createRpcRemote(contact)
+        try {
+            await rpcRemote.replicateData({ entry: dataEntry })
+        } catch (e) {
+            logger.trace('replicateData() threw an exception ' + e)
+        }
+    }
+
+    public async storeDataToDht(key: DhtAddress, data: Any, creator: DhtAddress): Promise<PeerDescriptor[]> {
+        logger.debug(`Storing data to DHT ${this.config.serviceId}`)
+        const result = await this.config.recursiveOperationManager.execute(key, RecursiveOperation.FIND_CLOSEST_NODES)
+        const closestNodes = result.closestNodes
+        const successfulNodes: PeerDescriptor[] = []
+        const ttl = this.config.highestTtl // ToDo: make TTL decrease according to some nice curve
+        const createdAt = Timestamp.now()
+        for (let i = 0; i < closestNodes.length && successfulNodes.length < this.config.redundancyFactor; i++) {
+            const keyRaw = getRawFromDhtAddress(key)
+            const creatorRaw = getRawFromDhtAddress(creator)
+            if (areEqualPeerDescriptors(this.config.localPeerDescriptor, closestNodes[i])) {
+                this.config.localDataStore.storeEntry({
+                    key: keyRaw,
+                    data,
+                    creator: creatorRaw,
+                    createdAt,
+                    storedAt: Timestamp.now(), 
+                    ttl, 
+                    stale: false,
+                    deleted: false,
+                })
+                successfulNodes.push(closestNodes[i])
+                continue
+            }
+            const rpcRemote = this.config.createRpcRemote(closestNodes[i])
+            try {
+                await rpcRemote.storeData({
+                    key: keyRaw,
+                    data,
+                    creator: creatorRaw,
+                    createdAt,
+                    ttl
+                })
+                successfulNodes.push(closestNodes[i])
+                logger.trace('remote.storeData() success')
+            } catch (e) {
+                logger.trace('remote.storeData() threw an exception ' + e)
+            }
+        }
+        return successfulNodes
+    }
+
+    private selfIsWithinRedundancyFactor(dataKey: DhtAddress): boolean {
+        const closestNeighbors = this.config.getClosestNeighborsTo(dataKey, this.config.redundancyFactor)
+        if (closestNeighbors.length < this.config.redundancyFactor) {
+            return true
+        } else {
+            const furthestCloseNeighbor = closestNeighbors[closestNeighbors.length - 1]
+            const dataKeyRaw = getRawFromDhtAddress(dataKey)
+            return getDistance(dataKeyRaw, this.config.localPeerDescriptor.nodeId) < getDistance(dataKeyRaw, furthestCloseNeighbor.nodeId)
+        }
+    }
+
+    private async replicateDataToClosestNodes(): Promise<void> {
+        const dataEntries = Array.from(this.config.localDataStore.values())
+        await Promise.all(dataEntries.map(async (dataEntry) => {
+            const neighbors = this.config.getClosestNeighborsTo(getDhtAddressFromRaw(dataEntry.key), this.config.redundancyFactor)
+            await Promise.all(neighbors.map(async (neighbor) => {
+                const rpcRemote = this.config.createRpcRemote(neighbor)
+                try {
+                    await rpcRemote.replicateData({ entry: dataEntry })
+                } catch (err) {
+                    logger.trace('Failed to replicate data in replicateDataToClosestNodes', { error: err })
+                }
+            }))
+        }))
+    }
+
+    private replicateDataToNeighbors(incomingPeer: PeerDescriptor, dataEntry: DataEntry): void {
+        // sort own contact list according to data id
+        const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
+        const incomingNodeId = getNodeIdFromPeerDescriptor(incomingPeer)
+        const key = getDhtAddressFromRaw(dataEntry.key)
+        // TODO use config option or named constant?
+        const closestToData = this.config.getClosestNeighborsTo(key, 10)
+        const sortedList = new SortedContactList<Contact>({
+            referenceId: key, 
+            maxSize: this.config.redundancyFactor, 
+            allowToContainReferenceId: true, 
+            emitEvents: false
+        })
+        sortedList.addContact(new Contact(this.config.localPeerDescriptor))
+        closestToData.forEach((neighbor) => {
+            sortedList.addContact(new Contact(neighbor))
+        })
+        const selfIsPrimaryStorer = (sortedList.getClosestContactId() === localNodeId)
+        const targets = selfIsPrimaryStorer
+            // if we are the closest to the data, replicate to all storageRedundancyFactor nearest
+            ? sortedList.getAllContacts()
+            // if we are not the closest node to the data, replicate only to the closest one to the data
+            : [sortedList.getAllContacts()[0]]
+        targets.forEach((contact) => {
+            const contactNodeId = contact.getNodeId()
+            if ((incomingNodeId !== contactNodeId) && (localNodeId !== contactNodeId)) {
+                setImmediate(() => {
+                    executeSafePromise(async () => {
+                        await this.replicateDataToContact(dataEntry, contact.getPeerDescriptor())
+                        logger.trace('replicateDataToContact() returned', { 
+                            node: contactNodeId,
+                            replicateOnlyToClosest: !selfIsPrimaryStorer
+                        })
+                    })
+                })
+            }
+        })
+    }
+
+    async destroy(): Promise<void> {
+        await this.replicateDataToClosestNodes()
+    }
+}

package/src/dht/store/StoreRpcLocal.ts

@@ -1,335 +1,66 @@
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { Logger } from '@streamr/utils'
+import { Empty } from '../../proto/google/protobuf/empty'
+import { Timestamp } from '../../proto/google/protobuf/timestamp'
 import {
-    DataEntry, DeleteDataRequest, DeleteDataResponse, MigrateDataRequest, MigrateDataResponse, PeerDescriptor,
+    DataEntry,
+    PeerDescriptor,
+    ReplicateDataRequest,
     StoreDataRequest, StoreDataResponse
 } from '../../proto/packages/dht/protos/DhtRpc'
-import { PeerID } from '../../helpers/PeerID'
-import { Any } from '../../proto/google/protobuf/any'
-import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { IStoreRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
 import { DhtCallContext } from '../../rpc-protocol/DhtCallContext'
-import { toProtoRpcClient } from '@streamr/proto-rpc'
-import { StoreRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
-import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
-import { IFinder } from '../find/Finder'
-import { areEqualPeerDescriptors, getNodeIdFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
-import { Logger } from '@streamr/utils'
 import { LocalDataStore } from './LocalDataStore'
-import { IStoreRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
-import { StoreRpcRemote } from './StoreRpcRemote'
-import { Timestamp } from '../../proto/google/protobuf/timestamp'
-import EventEmitter from 'eventemitter3'
-import { Events } from '../DhtNode'
-import { SortedContactList } from '../contact/SortedContactList'
-import { Contact } from '../contact/Contact'
-import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
-import { ServiceID } from '../../types/ServiceID'
+import { DhtAddress, getDhtAddressFromRaw } from '../../identifiers'
 
-interface DataStoreConfig {
-    rpcCommunicator: RoutingRpcCommunicator
-    finder: IFinder
-    localPeerDescriptor: PeerDescriptor
+interface StoreRpcLocalConfig {
     localDataStore: LocalDataStore
-    serviceId: ServiceID
-    maxTtl: number
-    highestTtl: number
-    redundancyFactor: number
-    dhtNodeEmitter: EventEmitter<Events>
-    getNodesClosestToIdFromBucket: (id: Uint8Array, n?: number) => DhtNodeRpcRemote[]
-    rpcRequestTimeout?: number
+    replicateDataToNeighbors: (incomingPeer: PeerDescriptor, dataEntry: DataEntry) => void
+    selfIsWithinRedundancyFactor: (key: DhtAddress) => boolean
 }
 
 const logger = new Logger(module)
 
 export class StoreRpcLocal implements IStoreRpc {
 
-    private readonly rpcCommunicator: RoutingRpcCommunicator
-    private readonly finder: IFinder
-    private readonly localPeerDescriptor: PeerDescriptor
-    private readonly localDataStore: LocalDataStore
-    private readonly serviceId: ServiceID
-    private readonly maxTtl: number
-    private readonly highestTtl: number
-    private readonly redundancyFactor: number
-    private readonly dhtNodeEmitter: EventEmitter<Events>
-    private readonly getNodesClosestToIdFromBucket: (id: Uint8Array, n?: number) => DhtNodeRpcRemote[]
-    private readonly rpcRequestTimeout?: number
-
-    constructor(config: DataStoreConfig) {
-        this.rpcCommunicator = config.rpcCommunicator
-        this.finder = config.finder
-        this.localPeerDescriptor = config.localPeerDescriptor
-        this.localDataStore = config.localDataStore
-        this.serviceId = config.serviceId
-        this.maxTtl = config.maxTtl
-        this.highestTtl = config.highestTtl
-        this.redundancyFactor = config.redundancyFactor
-        this.dhtNodeEmitter = config.dhtNodeEmitter
-        this.rpcRequestTimeout = config.rpcRequestTimeout
-        this.getNodesClosestToIdFromBucket = config.getNodesClosestToIdFromBucket
-        this.rpcCommunicator.registerRpcMethod(StoreDataRequest, StoreDataResponse, 'storeData',
-            (request: StoreDataRequest, context: ServerCallContext) => this.storeData(request, context))
-        this.rpcCommunicator.registerRpcMethod(MigrateDataRequest, MigrateDataResponse, 'migrateData',
-            (request: MigrateDataRequest, context: ServerCallContext) => this.migrateData(request, context))
-        this.rpcCommunicator.registerRpcMethod(DeleteDataRequest, DeleteDataResponse, 'deleteData',
-            (request: DeleteDataRequest, context: ServerCallContext) => this.deleteData(request, context))
-
-        this.dhtNodeEmitter.on('newContact', (peerDescriptor: PeerDescriptor) => {
-            this.localDataStore.getStore().forEach((dataMap, _dataKey) => {
-                dataMap.forEach((dataEntry) => {
-                    if (this.shouldMigrateDataToNewNode(dataEntry.dataEntry, peerDescriptor)) {
-                        this.migrateDataToContact(dataEntry.dataEntry, peerDescriptor)
-                    }
-                })
-            })
-        })
-    }
-
-    private shouldMigrateDataToNewNode(dataEntry: DataEntry, newNode: PeerDescriptor): boolean {
-
-        const dataId = PeerID.fromValue(dataEntry.kademliaId)
-        const newNodeId = PeerID.fromValue(newNode.kademliaId)
-        const localPeerId = PeerID.fromValue(this.localPeerDescriptor.kademliaId)
+    private readonly config: StoreRpcLocalConfig
 
-        const closestToData = this.getNodesClosestToIdFromBucket(dataEntry.kademliaId, 10)
-
-        const sortedList = new SortedContactList<Contact>(dataId, 20, undefined, true)
-        sortedList.addContact(new Contact(this.localPeerDescriptor))
-
-        closestToData.forEach((con) => {
-            if (!newNodeId.equals(PeerID.fromValue(con.getPeerDescriptor().kademliaId))) {
-                sortedList.addContact(new Contact(con.getPeerDescriptor()))
-            }
-        })
-
-        if (!sortedList.getAllContacts()[0].getPeerId().equals(localPeerId)) {
-            // If we are not the closes node to the data, do not migrate
-            return false
-        }
-
-        const newPeerId = PeerID.fromValue(newNode.kademliaId)
-        sortedList.addContact(new Contact(newNode))
-
-        const sorted = sortedList.getAllContacts()
-
-        let index = 0
-
-        for (index = 0; index < sorted.length; index++) {
-            if (sorted[index].getPeerId().equals(newPeerId)) {
-                break
-            }
-        }
-
-        // if new node is within the storageRedundancyFactor closest nodes to the data
-        // do migrate data to it
-
-        if (index < this.redundancyFactor) {
-            this.localDataStore.setStale(dataId, dataEntry.storer!, false)
-            return true
-        } else {
-            this.localDataStore.setStale(dataId, dataEntry.storer!, true)
-            return false
-        }
+    constructor(config: StoreRpcLocalConfig) {
+        this.config = config
     }
 
-    private async migrateDataToContact(dataEntry: DataEntry, contact: PeerDescriptor, doNotConnect: boolean = false): Promise<void> {
-        const rpcRemote = new StoreRpcRemote(
-            this.localPeerDescriptor,
-            contact,
-            this.serviceId,
-            toProtoRpcClient(new StoreRpcClient(this.rpcCommunicator.getRpcClientTransport())),
-            this.rpcRequestTimeout
-        )
-        try {
-            const response = await rpcRemote.migrateData({ dataEntry }, doNotConnect)
-            if (response.error) {
-                logger.trace('migrateData() returned error: ' + response.error)
-            }
-        } catch (e) {
-            logger.trace('migrateData() threw an exception ' + e)
-        }
-    }
-
-    public async storeDataToDht(key: Uint8Array, data: Any): Promise<PeerDescriptor[]> {
-        logger.debug(`Storing data to DHT ${this.serviceId}`)
-        const result = await this.finder.startFind(key)
-        const closestNodes = result.closestNodes
-        const successfulNodes: PeerDescriptor[] = []
-        const ttl = this.highestTtl // ToDo: make TTL decrease according to some nice curve
-        const storerTime = Timestamp.now()
-        for (let i = 0; i < closestNodes.length && successfulNodes.length < this.redundancyFactor; i++) {
-            if (areEqualPeerDescriptors(this.localPeerDescriptor, closestNodes[i])) {
-                this.localDataStore.storeEntry({
-                    kademliaId: key, 
-                    storer: this.localPeerDescriptor,
-                    ttl, 
-                    storedAt: Timestamp.now(), 
-                    data,
-                    stale: false,
-                    deleted: false,
-                    storerTime
-                })
-                successfulNodes.push(closestNodes[i])
-                continue
-            }
-            const rpcRemote = new StoreRpcRemote(
-                this.localPeerDescriptor,
-                closestNodes[i],
-                this.serviceId,
-                toProtoRpcClient(new StoreRpcClient(this.rpcCommunicator.getRpcClientTransport())),
-                this.rpcRequestTimeout
-            )
-            try {
-                const response = await rpcRemote.storeData({ kademliaId: key, data, ttl, storerTime })
-                if (!response.error) {
-                    successfulNodes.push(closestNodes[i])
-                    logger.trace('remote.storeData() returned success')
-                } else {
-                    logger.trace('remote.storeData() returned error: ' + response.error)
-                }
-            } catch (e) {
-                logger.trace('remote.storeData() threw an exception ' + e)
-            }
-        }
-        return successfulNodes
-    }
-
-    private selfIsOneOfClosestPeers(dataId: Uint8Array): boolean {
-        const localPeerId = PeerID.fromValue(this.localPeerDescriptor.kademliaId)
-        const closestPeers = this.getNodesClosestToIdFromBucket(dataId, this.redundancyFactor)
-        const sortedList = new SortedContactList<Contact>(localPeerId, this.redundancyFactor, undefined, true)
-        sortedList.addContact(new Contact(this.localPeerDescriptor))
-        closestPeers.forEach((con) => sortedList.addContact(new Contact(con.getPeerDescriptor())))
-        return sortedList.getClosestContacts().some((node) => node.getPeerId().equals(localPeerId))
-    }
-
-    public async deleteDataFromDht(key: Uint8Array): Promise<void> {
-        logger.debug(`Deleting data from DHT ${this.serviceId}`)
-        const result = await this.finder.startFind(key)
-        const closestNodes = result.closestNodes
-        const successfulNodes: PeerDescriptor[] = []
-        for (let i = 0; i < closestNodes.length && successfulNodes.length < this.redundancyFactor; i++) {
-            if (areEqualPeerDescriptors(this.localPeerDescriptor, closestNodes[i])) {
-                this.localDataStore.markAsDeleted(key, peerIdFromPeerDescriptor(this.localPeerDescriptor))
-                successfulNodes.push(closestNodes[i])
-                continue
-            }
-            const rpcRemote = new StoreRpcRemote(
-                this.localPeerDescriptor,
-                closestNodes[i],
-                this.serviceId,
-                toProtoRpcClient(new StoreRpcClient(this.rpcCommunicator.getRpcClientTransport())),
-                this.rpcRequestTimeout
-            )
-            try {
-                const response = await rpcRemote.deleteData({ kademliaId: key })
-                if (response.deleted) {
-                    logger.trace('remote.deleteData() returned success')
-                } else {
-                    logger.trace('could not delete data from ' + getNodeIdFromPeerDescriptor(closestNodes[i]))
-                }
-                successfulNodes.push(closestNodes[i])
-            } catch (e) {
-                logger.trace('remote.deleteData() threw an exception ' + e)
-            }
-        }
-    }
-
-    // RPC service implementation
-    async storeData(request: StoreDataRequest, context: ServerCallContext): Promise<StoreDataResponse> {
-        const ttl = Math.min(request.ttl, this.maxTtl)
-        const { incomingSourceDescriptor } = context as DhtCallContext
-        const { kademliaId, data, storerTime } = request
-        this.localDataStore.storeEntry({ 
-            kademliaId, 
-            storer: incomingSourceDescriptor!, 
-            ttl,
+    async storeData(request: StoreDataRequest): Promise<StoreDataResponse> {
+        logger.trace('storeData()')
+        const key = getDhtAddressFromRaw(request.key)
+        const selfIsOneOfClosestPeers = this.config.selfIsWithinRedundancyFactor(key)
+        this.config.localDataStore.storeEntry({ 
+            key: request.key,
+            data: request.data,
+            creator: request.creator,
+            createdAt: request.createdAt,
             storedAt: Timestamp.now(),
-            storerTime,
-            data,
-            stale: !this.selfIsOneOfClosestPeers(kademliaId),
+            ttl: request.ttl,
+            stale: !selfIsOneOfClosestPeers,
             deleted: false
         })
-        
-        if (!this.selfIsOneOfClosestPeers(kademliaId)) {
-            this.localDataStore.setAllEntriesAsStale(PeerID.fromValue(kademliaId))
+        if (!selfIsOneOfClosestPeers) {
+            this.config.localDataStore.setAllEntriesAsStale(key)
         }
-
-        logger.trace('storeData()')
-        return StoreDataResponse.create()
-    }
-
-    // RPC service implementation
-    async deleteData(request: DeleteDataRequest, context: ServerCallContext): Promise<DeleteDataResponse> {
-        const { incomingSourceDescriptor } = context as DhtCallContext
-        const { kademliaId } = request
-        const deleted = this.localDataStore.markAsDeleted(kademliaId, peerIdFromPeerDescriptor(incomingSourceDescriptor!))
-        return DeleteDataResponse.create({ deleted })
+        return {}
     }
 
-    // RPC service implementation
-    public async migrateData(request: MigrateDataRequest, context: ServerCallContext): Promise<MigrateDataResponse> {
-        logger.trace('server-side migrateData()')
-        const dataEntry = request.dataEntry!
-
-        const wasStored = this.localDataStore.storeEntry(dataEntry)
-        
+    public async replicateData(request: ReplicateDataRequest, context: ServerCallContext): Promise<Empty> {
+        logger.trace('server-side replicateData()')
+        const dataEntry = request.entry!
+        const wasStored = this.config.localDataStore.storeEntry(dataEntry)
         if (wasStored) {
-            this.migrateDataToNeighborsIfNeeded((context as DhtCallContext).incomingSourceDescriptor!, request.dataEntry!)
+            this.config.replicateDataToNeighbors((context as DhtCallContext).incomingSourceDescriptor!, request.entry!)
         }
-        if (!this.selfIsOneOfClosestPeers(dataEntry.kademliaId)) {
-            this.localDataStore.setAllEntriesAsStale(PeerID.fromValue(dataEntry.kademliaId))
-        }
-        logger.trace('server-side migrateData() at end')
-        return MigrateDataResponse.create()
-    }
-
-    private migrateDataToNeighborsIfNeeded(incomingPeer: PeerDescriptor, dataEntry: DataEntry): void {
-
-        // sort own contact list according to data id
-        const localPeerId = PeerID.fromValue(this.localPeerDescriptor.kademliaId)
-        const dataId = PeerID.fromValue(dataEntry.kademliaId)
-        const incomingPeerId = PeerID.fromValue(incomingPeer.kademliaId)
-        const closestToData = this.getNodesClosestToIdFromBucket(dataEntry.kademliaId, 10)
-
-        const sortedList = new SortedContactList<Contact>(dataId, this.redundancyFactor, undefined, true)
-        sortedList.addContact(new Contact(this.localPeerDescriptor))
-
-        closestToData.forEach((con) => {
-            sortedList.addContact(new Contact(con.getPeerDescriptor()))
-        })
-
-        if (!sortedList.getAllContacts()[0].getPeerId().equals(localPeerId)) {
-            // If we are not the closest node to the data, migrate only to the 
-            // closest one to the data
-
-            const contact = sortedList.getAllContacts()[0]
-            const contactPeerId = PeerID.fromValue(contact.getPeerDescriptor().kademliaId)
-            if (!incomingPeerId.equals(contactPeerId) && !localPeerId.equals(contactPeerId)) {
-                setImmediate(async () => {
-                    try {
-                        await this.migrateDataToContact(dataEntry, contact.getPeerDescriptor())
-                        logger.trace('migrateDataToContact() returned when migrating to only the closest contact')
-                    } catch (e) {
-                        logger.error('migrating data to only the closest contact failed ' + e)
-                    }
-                })
-            }
-        } else {
-            // if we are the closest to the data, migrate to all storageRedundancyFactor nearest
-            sortedList.getAllContacts().forEach((contact) => {
-                const contactPeerId = PeerID.fromValue(contact.getPeerDescriptor().kademliaId)
-                if (!incomingPeerId.equals(contactPeerId) && !localPeerId.equals(contactPeerId)) {
-                    if (!incomingPeerId.equals(contactPeerId) && !localPeerId.equals(contactPeerId)) {
-                        setImmediate(async () => {
-                            try {
-                                await this.migrateDataToContact(dataEntry, contact.getPeerDescriptor())
-                                logger.trace('migrateDataToContact() returned')
-                            } catch (e) {
-                                logger.error('migrating data to one of the closest contacts failed ' + e)
-                            }
-                        })
-                    }
-                }
-            })
+        const key = getDhtAddressFromRaw(dataEntry.key)
+        if (!this.config.selfIsWithinRedundancyFactor(key)) {
+            this.config.localDataStore.setAllEntriesAsStale(key)
         }
+        logger.trace('server-side replicateData() at end')
+        return {}
     }
 }

package/src/dht/store/StoreRpcRemote.ts

@@ -1,47 +1,29 @@
-import { EXISTING_CONNECTION_TIMEOUT, Remote } from '../contact/Remote'
-import { IStoreRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
-import { 
-    DeleteDataRequest,
-    DeleteDataResponse,
-    MigrateDataRequest,
-    MigrateDataResponse,
-    StoreDataRequest,
-    StoreDataResponse
+import { getNodeIdFromPeerDescriptor } from '../../identifiers'
+import {
+    ReplicateDataRequest,
+    StoreDataRequest
 } from '../../proto/packages/dht/protos/DhtRpc'
-import { getNodeIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
+import { StoreRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { EXISTING_CONNECTION_TIMEOUT, RpcRemote } from '../contact/RpcRemote'
 
-export class StoreRpcRemote extends Remote<IStoreRpcClient> {
+export class StoreRpcRemote extends RpcRemote<StoreRpcClient> {
 
-    async storeData(request: StoreDataRequest): Promise<StoreDataResponse> {
+    async storeData(request: StoreDataRequest): Promise<void> {
         const options = this.formDhtRpcOptions()
         try {
-            return await this.getClient().storeData(request, options)
+            await this.getClient().storeData(request, options)
         } catch (err) {
             const to = getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
             const from = getNodeIdFromPeerDescriptor(this.getLocalPeerDescriptor())
-            throw Error(
-                `Could not store data to ${to} from ${from} ${err}`
-            )
+            throw new Error(`Could not store data to ${to} from ${from} ${err}`)
         }
     }
 
-    async deleteData(request: DeleteDataRequest): Promise<DeleteDataResponse> {
-        const options = this.formDhtRpcOptions()
-        try {
-            return await this.getClient().deleteData(request, options)
-        } catch (err) {
-            throw Error(
-                `Could not call delete data to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())} ${err}`
-            )
-        }
-    }
-
-    async migrateData(request: MigrateDataRequest, doNotConnect: boolean = false): Promise<MigrateDataResponse> {
+    async replicateData(request: ReplicateDataRequest): Promise<void> {
         const options = this.formDhtRpcOptions({
-            timeout: EXISTING_CONNECTION_TIMEOUT,
-            doNotConnect
+            timeout: EXISTING_CONNECTION_TIMEOUT
         })
-        return this.getClient().migrateData(request, options)
+        return this.getClient().replicateData(request, options)
     }
 
 }

package/src/exports.ts

@@ -7,13 +7,23 @@ export { getRandomRegion, getRegionDelayMatrix } from './connection/simulator/pi
 export { PeerDescriptor, Message, NodeType, DataEntry } from './proto/packages/dht/protos/DhtRpc'
 export { ITransport } from './transport/ITransport'
 export { ConnectionManager, ConnectionLocker, PortRange, TlsCertificate } from './connection/ConnectionManager'
+export { LockID } from './connection/ConnectionLockHandler'
 export { DefaultConnectorFacade } from './connection/ConnectorFacade'
 export { DhtRpcOptions } from './rpc-protocol/DhtRpcOptions'
-export { Remote, EXISTING_CONNECTION_TIMEOUT } from './dht/contact/Remote'
-export { areEqualPeerDescriptors } from './helpers/peerIdFromPeerDescriptor'
+export { RpcRemote, EXISTING_CONNECTION_TIMEOUT } from './dht/contact/RpcRemote'
 export { IceServer } from './connection/webrtc/WebrtcConnector'
 export { DhtCallContext } from './rpc-protocol/DhtCallContext'
 export { ClientWebsocket } from './connection/websocket/ClientWebsocket'
 export { ManagedConnection } from './connection/ManagedConnection'
 export { ConnectionType } from './connection/IConnection'
-export { PeerID } from './helpers/PeerID'
+export { ServiceID } from './types/ServiceID'
+export { RingContacts } from './dht/contact/RingContactList'
+export { 
+    DhtAddress,
+    DhtAddressRaw,
+    getDhtAddressFromRaw,
+    getRawFromDhtAddress,
+    createRandomDhtAddress,
+    areEqualPeerDescriptors,
+    getNodeIdFromPeerDescriptor
+} from './identifiers'