@streamr/trackerless-network 103.3.1 → 103.6.0-rc.0

package/dist/exports.cjs

@@ -29,7 +29,7 @@ class ExternalNetworkRpc {
     }
 }
 
-var version = "103.3.1";
+var version = "103.6.0-rc.0";
 
 // @generated message type with reflection information, may provide speed optimized methods
 class Any$Type extends runtime.MessageType {
@@ -1676,6 +1676,18 @@ class PauseNeighborRequest$Type extends runtime.MessageType {
  */
 const PauseNeighborRequest = new PauseNeighborRequest$Type();
 // @generated message type with reflection information, may provide speed optimized methods
+class PauseNeighborResponse$Type extends runtime.MessageType {
+    constructor() {
+        super("PauseNeighborResponse", [
+            { no: 1, name: "accepted", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
+        ]);
+    }
+}
+/**
+ * @generated MessageType for protobuf message PauseNeighborResponse
+ */
+const PauseNeighborResponse = new PauseNeighborResponse$Type();
+// @generated message type with reflection information, may provide speed optimized methods
 class ResumeNeighborRequest$Type extends runtime.MessageType {
     constructor() {
         super("ResumeNeighborRequest", [
@@ -1731,7 +1743,7 @@ const NodeInfoRpc = new runtimeRpc.ServiceType("NodeInfoRpc", [
  * @generated ServiceType for protobuf service PlumtreeRpc
  */
 const PlumtreeRpc = new runtimeRpc.ServiceType("PlumtreeRpc", [
-    { name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: Empty },
+    { name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: PauseNeighborResponse },
     { name: "resumeNeighbor", options: {}, I: ResumeNeighborRequest, O: Empty },
     { name: "sendMetadata", options: {}, I: MessageID, O: Empty }
 ]);
@@ -3333,14 +3345,17 @@ class PlumtreeRpcLocal {
     async pauseNeighbor(request, context) {
         const sender = dht.toNodeId(context.incomingSourceDescriptor);
         if (this.neighbors.has(sender)) {
-            this.pausedNodes.add(sender, request.messageChainId);
+            const accepted = this.pausedNodes.add(sender, request.messageChainId);
+            return { accepted };
         }
-        return Empty;
+        return { accepted: false };
     }
     async resumeNeighbor(request, context) {
         const sender = context.incomingSourceDescriptor;
-        this.pausedNodes.delete(dht.toNodeId(sender), request.messageChainId);
-        await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
+        if (this.neighbors.has(dht.toNodeId(sender))) {
+            this.pausedNodes.delete(dht.toNodeId(sender), request.messageChainId);
+            await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
+        }
         return Empty;
     }
 }
@@ -3353,10 +3368,9 @@ class PlumtreeRpcRemote extends dht.RpcRemote {
         await this.getClient().sendMetadata(msg, options);
     }
     async pauseNeighbor(messageChainId) {
-        const options = this.formDhtRpcOptions({
-            notification: true
-        });
-        await this.getClient().pauseNeighbor({ messageChainId }, options);
+        const options = this.formDhtRpcOptions();
+        const response = await this.getClient().pauseNeighbor({ messageChainId }, options);
+        return response.accepted;
     }
     async resumeNeighbor(fromTimestamp, messageChainId) {
         const options = this.formDhtRpcOptions({
@@ -3378,9 +3392,10 @@ class PausedNeighbors {
             this.pausedNeighbors.set(msgChainId, new Set());
         }
         if (this.pausedNeighbors.get(msgChainId).size >= this.limit) {
-            return;
+            return false;
         }
         this.pausedNeighbors.get(msgChainId).add(node);
+        return true;
     }
     delete(node, msgChainId) {
         this.pausedNeighbors.get(msgChainId)?.delete(node);
@@ -3413,19 +3428,24 @@ class PausedNeighbors {
 }
 
 const MAX_PAUSED_NEIGHBORS_DEFAULT = 3;
+const DEFAULT_RECOVERY_TIMEOUT = 500;
+const DEFAULT_RECOVERY_CHECK_INTERVAL = 200;
+const DEFAULT_RECOVERY_COOLDOWN = 2500;
 const logger$4 = new utils.Logger('PlumtreeManager');
 class PlumtreeManager extends eventemitter3.EventEmitter {
     neighbors;
     localPeerDescriptor;
-    // We have paused sending real data to these neighbrs and only send metadata
     localPausedNeighbors;
-    // We have asked these nodes to pause sending real data to us, used to limit sending of pausing and resuming requests
     remotePausedNeighbors;
     rpcLocal;
     latestMessages = new Map();
     rpcCommunicator;
-    metadataTimestampsAheadOfRealData = new Map();
     maxPausedNeighbors;
+    recoveryState = new Map();
+    recoveryCooldownUntil = new Map();
+    recoveryTimeout;
+    recoveryCooldown;
+    abortController = new AbortController();
     constructor(options) {
         super();
         this.neighbors = options.neighbors;
@@ -3433,12 +3453,22 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
         this.localPeerDescriptor = options.localPeerDescriptor;
         this.localPausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
         this.remotePausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
+        this.recoveryTimeout = options.recoveryTimeout ?? DEFAULT_RECOVERY_TIMEOUT;
+        this.recoveryCooldown = options.recoveryCooldown ?? DEFAULT_RECOVERY_COOLDOWN;
         this.rpcLocal = new PlumtreeRpcLocal(this.neighbors, this.localPausedNeighbors, (metadata, previousNode) => this.onMetadata(metadata, previousNode), (fromTimestamp, msgChainId, remotePeerDescriptor) => this.sendBuffer(fromTimestamp, msgChainId, remotePeerDescriptor));
-        this.neighbors.on('nodeRemoved', (nodeId) => this.onNeighborRemoved(nodeId));
+        this.neighbors.on('nodeRemoved', this.onNeighborRemoved);
         this.rpcCommunicator = options.rpcCommunicator;
         this.rpcCommunicator.registerRpcNotification(MessageID, 'sendMetadata', (msg, context) => this.rpcLocal.sendMetadata(msg, context));
-        this.rpcCommunicator.registerRpcNotification(PauseNeighborRequest, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
+        this.rpcCommunicator.registerRpcMethod(PauseNeighborRequest, PauseNeighborResponse, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
         this.rpcCommunicator.registerRpcNotification(ResumeNeighborRequest, 'resumeNeighbor', (msg, context) => this.rpcLocal.resumeNeighbor(msg, context));
+        utils.setAbortableInterval(() => {
+            const now = performance.now();
+            for (const [chainId, state] of this.recoveryState) {
+                if (now - state.metadataAheadSince >= this.recoveryTimeout && !state.resumeInProgress) {
+                    this.attemptRecovery(chainId, state, this.getLatestMessageTimestamp(chainId));
+                }
+            }
+        }, options.recoveryCheckInterval ?? DEFAULT_RECOVERY_CHECK_INTERVAL, this.abortController.signal);
     }
     async pauseNeighbor(node, msgChainId) {
         if (this.neighbors.has(dht.toNodeId(node))
@@ -3446,8 +3476,16 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
             && this.remotePausedNeighbors.size(msgChainId) < this.maxPausedNeighbors) {
             logger$4.debug(`Pausing neighbor ${dht.toNodeId(node)}`);
             this.remotePausedNeighbors.add(dht.toNodeId(node), msgChainId);
-            const remote = this.createRemote(node);
-            await remote.pauseNeighbor(msgChainId);
+            try {
+                const remote = this.createRemote(node);
+                const accepted = await remote.pauseNeighbor(msgChainId);
+                if (!accepted) {
+                    this.remotePausedNeighbors.delete(dht.toNodeId(node), msgChainId);
+                }
+            }
+            catch (_e) {
+                this.remotePausedNeighbors.delete(dht.toNodeId(node), msgChainId);
+            }
         }
     }
     async resumeNeighbor(node, msgChainId, fromTimestamp) {
@@ -3458,9 +3496,15 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
             await remote.resumeNeighbor(fromTimestamp, msgChainId);
         }
     }
-    onNeighborRemoved(nodeId) {
+    onNeighborRemoved = (nodeId) => {
         this.localPausedNeighbors.deleteAll(nodeId);
         this.remotePausedNeighbors.deleteAll(nodeId);
+        for (const [_chainId, state] of this.recoveryState) {
+            state.candidates = state.candidates.filter((c) => dht.toNodeId(c) !== nodeId);
+            if (state.lastAttemptedNode !== null && dht.toNodeId(state.lastAttemptedNode) === nodeId) {
+                state.lastAttemptedNode = null;
+            }
+        }
         if (this.neighbors.size() > 0) {
             this.remotePausedNeighbors.forEach((pausedNeighbors, msgChainId) => {
                 if (pausedNeighbors.size >= this.neighbors.size()) {
@@ -3470,7 +3514,7 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
                 }
             });
         }
-    }
+    };
     getLatestMessageTimestamp(msgChainId) {
         if (!this.latestMessages.has(msgChainId) || this.latestMessages.get(msgChainId).length === 0) {
             return 0;
@@ -3480,22 +3524,61 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
     async sendBuffer(fromTimestamp, msgChainId, neighbor) {
         const remote = new ContentDeliveryRpcRemote(this.localPeerDescriptor, neighbor, this.rpcCommunicator, ContentDeliveryRpcClient);
         const messages = this.latestMessages.get(msgChainId)?.filter((msg) => msg.messageId.timestamp > fromTimestamp) ?? [];
-        await Promise.all(messages.map((msg) => remote.sendStreamMessage(msg)));
+        for (const msg of messages) {
+            await remote.sendStreamMessage(msg);
+        }
     }
     async onMetadata(msg, previousNode) {
-        // If we receive newer metadata than messages in the buffer, resume the sending neighbor
-        const latestMessageTimestamp = this.getLatestMessageTimestamp(msg.messageChainId);
-        if (latestMessageTimestamp < msg.timestamp) {
-            if (!this.metadataTimestampsAheadOfRealData.has(msg.messageChainId)) {
-                this.metadataTimestampsAheadOfRealData.set(msg.messageChainId, new Set());
-            }
-            this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).add(msg.timestamp);
-            if (this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).size > 1) {
-                await this.resumeNeighbor(previousNode, msg.messageChainId, this.getLatestMessageTimestamp(msg.messageChainId));
-                this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).forEach((timestamp) => {
-                    this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).delete(timestamp);
-                });
-            }
+        const latestTs = this.getLatestMessageTimestamp(msg.messageChainId);
+        if (latestTs >= msg.timestamp) {
+            return;
+        }
+        const chainId = msg.messageChainId;
+        const cooldownUntil = this.recoveryCooldownUntil.get(chainId);
+        if (cooldownUntil !== undefined && performance.now() < cooldownUntil) {
+            return;
+        }
+        let state = this.recoveryState.get(chainId);
+        if (!state) {
+            state = {
+                timestampsAhead: new Set(),
+                metadataAheadSince: performance.now(),
+                candidates: [],
+                lastAttemptedNode: null,
+                resumeInProgress: false
+            };
+            this.recoveryState.set(chainId, state);
+        }
+        state.timestampsAhead.add(msg.timestamp);
+        const nodeId = dht.toNodeId(previousNode);
+        const isLastAttempted = state.lastAttemptedNode !== null && dht.toNodeId(state.lastAttemptedNode) === nodeId;
+        if (!isLastAttempted && !state.candidates.some((c) => dht.toNodeId(c) === nodeId)) {
+            state.candidates.push(previousNode);
+        }
+        if (state.timestampsAhead.size > 1 && !state.resumeInProgress) {
+            await this.attemptRecovery(chainId, state, latestTs);
+        }
+    }
+    async attemptRecovery(chainId, state, latestTs) {
+        const candidate = state.candidates.shift();
+        if (!candidate) {
+            state.metadataAheadSince = performance.now();
+            return;
+        }
+        state.resumeInProgress = true;
+        state.lastAttemptedNode = candidate;
+        state.candidates = [];
+        state.timestampsAhead.clear();
+        state.metadataAheadSince = performance.now();
+        try {
+            const remote = this.createRemote(candidate);
+            await remote.resumeNeighbor(latestTs, chainId);
+        }
+        catch (_e) {
+            logger$4.debug('Recovery resume failed, will retry with next candidate');
+        }
+        finally {
+            state.resumeInProgress = false;
         }
     }
     createRemote(neighbor) {
@@ -3513,8 +3596,13 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
             this.latestMessages.get(messageChainId).shift();
             this.latestMessages.get(messageChainId).push(msg);
         }
-        if (this.metadataTimestampsAheadOfRealData.has(msg.messageId.messageChainId)) {
-            this.metadataTimestampsAheadOfRealData.get(msg.messageId.messageChainId).delete(msg.messageId.timestamp);
+        const state = this.recoveryState.get(messageChainId);
+        if (state) {
+            if (state.lastAttemptedNode) {
+                this.remotePausedNeighbors.delete(dht.toNodeId(state.lastAttemptedNode), messageChainId);
+            }
+            this.recoveryState.delete(messageChainId);
+            this.recoveryCooldownUntil.set(messageChainId, performance.now() + this.recoveryCooldown);
         }
         this.emit('message', msg);
         const neighbors = this.neighbors.getAll().filter((neighbor) => dht.toNodeId(neighbor.getPeerDescriptor()) !== previousNode);
@@ -3532,7 +3620,14 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
         return this.localPausedNeighbors.isPaused(dht.toNodeId(node), msgChainId)
             || this.remotePausedNeighbors.isPaused(dht.toNodeId(node), msgChainId);
     }
+    getLocalPausedNeighbors() {
+        return this.localPausedNeighbors;
+    }
+    getRemotePausedNeighbors() {
+        return this.remotePausedNeighbors;
+    }
     stop() {
+        this.abortController.abort();
         this.neighbors.off('nodeRemoved', this.onNeighborRemoved);
     }
 }