@streamr/trackerless-network 103.3.0 → 103.6.0-rc.0

package/dist/exports.d.ts

@@ -1084,11 +1084,26 @@ type ContentDeliveryLayerNeighborInfo = MarkRequired<ContentDeliveryLayerNeighbo
 type StreamPartitionInfo = ChangeFieldType<Required<StreamPartitionInfo$1>, 'contentDeliveryLayerNeighbors', ContentDeliveryLayerNeighborInfo[]>;
 type NodeInfo = ChangeFieldType<Required<NodeInfoResponse>, 'streamPartitions', StreamPartitionInfo[]>;
 
+declare class PausedNeighbors {
+    private readonly pausedNeighbors;
+    private readonly limit;
+    constructor(limit: number);
+    add(node: DhtAddress, msgChainId: string): boolean;
+    delete(node: DhtAddress, msgChainId: string): void;
+    deleteAll(node: DhtAddress): void;
+    isPaused(node: DhtAddress, msgChainId: string): boolean;
+    forEach(fn: (neighbors: Set<DhtAddress>, msgChainId: string) => void): void;
+    size(msgChainId: string): number;
+}
+
 interface Options {
     neighbors: NodeList;
     localPeerDescriptor: PeerDescriptor$1;
     rpcCommunicator: ListeningRpcCommunicator;
     maxPausedNeighbors?: number;
+    recoveryTimeout?: number;
+    recoveryCheckInterval?: number;
+    recoveryCooldown?: number;
 }
 interface Events$3 {
     message: (msg: StreamMessage) => void;
@@ -1101,8 +1116,12 @@ declare class PlumtreeManager extends EventEmitter<Events$3> {
     private readonly rpcLocal;
     private readonly latestMessages;
     private readonly rpcCommunicator;
-    private readonly metadataTimestampsAheadOfRealData;
     private readonly maxPausedNeighbors;
+    private readonly recoveryState;
+    private readonly recoveryCooldownUntil;
+    private readonly recoveryTimeout;
+    private readonly recoveryCooldown;
+    private readonly abortController;
     constructor(options: Options);
     pauseNeighbor(node: PeerDescriptor$1, msgChainId: string): Promise<void>;
     resumeNeighbor(node: PeerDescriptor$1, msgChainId: string, fromTimestamp: number): Promise<void>;
@@ -1110,9 +1129,12 @@ declare class PlumtreeManager extends EventEmitter<Events$3> {
     getLatestMessageTimestamp(msgChainId: string): number;
     private sendBuffer;
     private onMetadata;
+    private attemptRecovery;
     private createRemote;
     broadcast(msg: StreamMessage, previousNode: DhtAddress): void;
     isNeighborPaused(node: PeerDescriptor$1, msgChainId: string): boolean;
+    getLocalPausedNeighbors(): PausedNeighbors;
+    getRemotePausedNeighbors(): PausedNeighbors;
     stop(): void;
 }
 

package/dist/exports.js

@@ -1,6 +1,6 @@
 import { ListeningRpcCommunicator, PeerDescriptor as PeerDescriptor$1, areEqualPeerDescriptors, toNodeId, RpcRemote, toDhtAddress, toDhtAddressRaw, EXISTING_CONNECTION_TIMEOUT, DhtNode } from '@streamr/dht';
 import { toProtoRpcClient } from '@streamr/proto-rpc';
-import { Logger, scheduleAtInterval, wait, setAbortableTimeout, toUserId, addManagedEventListener, waitForEvent, toUserIdRaw, computeSha1, MetricsContext, RateMetric, toStreamPartID, StreamPartIDUtils, until } from '@streamr/utils';
+import { Logger, scheduleAtInterval, wait, setAbortableTimeout, toUserId, addManagedEventListener, waitForEvent, setAbortableInterval, toUserIdRaw, computeSha1, MetricsContext, RateMetric, toStreamPartID, StreamPartIDUtils, until } from '@streamr/utils';
 import pull from 'lodash/pull';
 import { EventEmitter } from 'eventemitter3';
 import sampleSize from 'lodash/sampleSize';
@@ -27,7 +27,7 @@ class ExternalNetworkRpc {
     }
 }
 
-var version = "103.3.0";
+var version = "103.6.0-rc.0";
 
 // @generated message type with reflection information, may provide speed optimized methods
 class Any$Type extends MessageType {
@@ -1674,6 +1674,18 @@ class PauseNeighborRequest$Type extends MessageType {
  */
 const PauseNeighborRequest = new PauseNeighborRequest$Type();
 // @generated message type with reflection information, may provide speed optimized methods
+class PauseNeighborResponse$Type extends MessageType {
+    constructor() {
+        super("PauseNeighborResponse", [
+            { no: 1, name: "accepted", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
+        ]);
+    }
+}
+/**
+ * @generated MessageType for protobuf message PauseNeighborResponse
+ */
+const PauseNeighborResponse = new PauseNeighborResponse$Type();
+// @generated message type with reflection information, may provide speed optimized methods
 class ResumeNeighborRequest$Type extends MessageType {
     constructor() {
         super("ResumeNeighborRequest", [
@@ -1729,7 +1741,7 @@ const NodeInfoRpc = new ServiceType("NodeInfoRpc", [
  * @generated ServiceType for protobuf service PlumtreeRpc
  */
 const PlumtreeRpc = new ServiceType("PlumtreeRpc", [
-    { name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: Empty },
+    { name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: PauseNeighborResponse },
     { name: "resumeNeighbor", options: {}, I: ResumeNeighborRequest, O: Empty },
     { name: "sendMetadata", options: {}, I: MessageID, O: Empty }
 ]);
@@ -3331,14 +3343,17 @@ class PlumtreeRpcLocal {
     async pauseNeighbor(request, context) {
         const sender = toNodeId(context.incomingSourceDescriptor);
         if (this.neighbors.has(sender)) {
-            this.pausedNodes.add(sender, request.messageChainId);
+            const accepted = this.pausedNodes.add(sender, request.messageChainId);
+            return { accepted };
         }
-        return Empty;
+        return { accepted: false };
     }
     async resumeNeighbor(request, context) {
         const sender = context.incomingSourceDescriptor;
-        this.pausedNodes.delete(toNodeId(sender), request.messageChainId);
-        await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
+        if (this.neighbors.has(toNodeId(sender))) {
+            this.pausedNodes.delete(toNodeId(sender), request.messageChainId);
+            await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
+        }
         return Empty;
     }
 }
@@ -3351,10 +3366,9 @@ class PlumtreeRpcRemote extends RpcRemote {
         await this.getClient().sendMetadata(msg, options);
     }
     async pauseNeighbor(messageChainId) {
-        const options = this.formDhtRpcOptions({
-            notification: true
-        });
-        await this.getClient().pauseNeighbor({ messageChainId }, options);
+        const options = this.formDhtRpcOptions();
+        const response = await this.getClient().pauseNeighbor({ messageChainId }, options);
+        return response.accepted;
     }
     async resumeNeighbor(fromTimestamp, messageChainId) {
         const options = this.formDhtRpcOptions({
@@ -3376,9 +3390,10 @@ class PausedNeighbors {
             this.pausedNeighbors.set(msgChainId, new Set());
         }
         if (this.pausedNeighbors.get(msgChainId).size >= this.limit) {
-            return;
+            return false;
         }
         this.pausedNeighbors.get(msgChainId).add(node);
+        return true;
     }
     delete(node, msgChainId) {
         this.pausedNeighbors.get(msgChainId)?.delete(node);
@@ -3411,19 +3426,24 @@ class PausedNeighbors {
 }
 
 const MAX_PAUSED_NEIGHBORS_DEFAULT = 3;
+const DEFAULT_RECOVERY_TIMEOUT = 500;
+const DEFAULT_RECOVERY_CHECK_INTERVAL = 200;
+const DEFAULT_RECOVERY_COOLDOWN = 2500;
 const logger$4 = new Logger('PlumtreeManager');
 class PlumtreeManager extends EventEmitter {
     neighbors;
     localPeerDescriptor;
-    // We have paused sending real data to these neighbrs and only send metadata
     localPausedNeighbors;
-    // We have asked these nodes to pause sending real data to us, used to limit sending of pausing and resuming requests
     remotePausedNeighbors;
     rpcLocal;
     latestMessages = new Map();
     rpcCommunicator;
-    metadataTimestampsAheadOfRealData = new Map();
     maxPausedNeighbors;
+    recoveryState = new Map();
+    recoveryCooldownUntil = new Map();
+    recoveryTimeout;
+    recoveryCooldown;
+    abortController = new AbortController();
     constructor(options) {
         super();
         this.neighbors = options.neighbors;
@@ -3431,12 +3451,22 @@ class PlumtreeManager extends EventEmitter {
         this.localPeerDescriptor = options.localPeerDescriptor;
         this.localPausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
         this.remotePausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
+        this.recoveryTimeout = options.recoveryTimeout ?? DEFAULT_RECOVERY_TIMEOUT;
+        this.recoveryCooldown = options.recoveryCooldown ?? DEFAULT_RECOVERY_COOLDOWN;
         this.rpcLocal = new PlumtreeRpcLocal(this.neighbors, this.localPausedNeighbors, (metadata, previousNode) => this.onMetadata(metadata, previousNode), (fromTimestamp, msgChainId, remotePeerDescriptor) => this.sendBuffer(fromTimestamp, msgChainId, remotePeerDescriptor));
-        this.neighbors.on('nodeRemoved', (nodeId) => this.onNeighborRemoved(nodeId));
+        this.neighbors.on('nodeRemoved', this.onNeighborRemoved);
         this.rpcCommunicator = options.rpcCommunicator;
         this.rpcCommunicator.registerRpcNotification(MessageID, 'sendMetadata', (msg, context) => this.rpcLocal.sendMetadata(msg, context));
-        this.rpcCommunicator.registerRpcNotification(PauseNeighborRequest, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
+        this.rpcCommunicator.registerRpcMethod(PauseNeighborRequest, PauseNeighborResponse, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
         this.rpcCommunicator.registerRpcNotification(ResumeNeighborRequest, 'resumeNeighbor', (msg, context) => this.rpcLocal.resumeNeighbor(msg, context));
+        setAbortableInterval(() => {
+            const now = performance.now();
+            for (const [chainId, state] of this.recoveryState) {
+                if (now - state.metadataAheadSince >= this.recoveryTimeout && !state.resumeInProgress) {
+                    this.attemptRecovery(chainId, state, this.getLatestMessageTimestamp(chainId));
+                }
+            }
+        }, options.recoveryCheckInterval ?? DEFAULT_RECOVERY_CHECK_INTERVAL, this.abortController.signal);
     }
     async pauseNeighbor(node, msgChainId) {
         if (this.neighbors.has(toNodeId(node))
@@ -3444,8 +3474,16 @@ class PlumtreeManager extends EventEmitter {
             && this.remotePausedNeighbors.size(msgChainId) < this.maxPausedNeighbors) {
             logger$4.debug(`Pausing neighbor ${toNodeId(node)}`);
             this.remotePausedNeighbors.add(toNodeId(node), msgChainId);
-            const remote = this.createRemote(node);
-            await remote.pauseNeighbor(msgChainId);
+            try {
+                const remote = this.createRemote(node);
+                const accepted = await remote.pauseNeighbor(msgChainId);
+                if (!accepted) {
+                    this.remotePausedNeighbors.delete(toNodeId(node), msgChainId);
+                }
+            }
+            catch (_e) {
+                this.remotePausedNeighbors.delete(toNodeId(node), msgChainId);
+            }
         }
     }
     async resumeNeighbor(node, msgChainId, fromTimestamp) {
@@ -3456,9 +3494,15 @@ class PlumtreeManager extends EventEmitter {
             await remote.resumeNeighbor(fromTimestamp, msgChainId);
         }
     }
-    onNeighborRemoved(nodeId) {
+    onNeighborRemoved = (nodeId) => {
         this.localPausedNeighbors.deleteAll(nodeId);
         this.remotePausedNeighbors.deleteAll(nodeId);
+        for (const [_chainId, state] of this.recoveryState) {
+            state.candidates = state.candidates.filter((c) => toNodeId(c) !== nodeId);
+            if (state.lastAttemptedNode !== null && toNodeId(state.lastAttemptedNode) === nodeId) {
+                state.lastAttemptedNode = null;
+            }
+        }
         if (this.neighbors.size() > 0) {
             this.remotePausedNeighbors.forEach((pausedNeighbors, msgChainId) => {
                 if (pausedNeighbors.size >= this.neighbors.size()) {
@@ -3468,7 +3512,7 @@ class PlumtreeManager extends EventEmitter {
                 }
             });
         }
-    }
+    };
     getLatestMessageTimestamp(msgChainId) {
         if (!this.latestMessages.has(msgChainId) || this.latestMessages.get(msgChainId).length === 0) {
             return 0;
@@ -3478,22 +3522,61 @@ class PlumtreeManager extends EventEmitter {
     async sendBuffer(fromTimestamp, msgChainId, neighbor) {
         const remote = new ContentDeliveryRpcRemote(this.localPeerDescriptor, neighbor, this.rpcCommunicator, ContentDeliveryRpcClient);
         const messages = this.latestMessages.get(msgChainId)?.filter((msg) => msg.messageId.timestamp > fromTimestamp) ?? [];
-        await Promise.all(messages.map((msg) => remote.sendStreamMessage(msg)));
+        for (const msg of messages) {
+            await remote.sendStreamMessage(msg);
+        }
     }
     async onMetadata(msg, previousNode) {
-        // If we receive newer metadata than messages in the buffer, resume the sending neighbor
-        const latestMessageTimestamp = this.getLatestMessageTimestamp(msg.messageChainId);
-        if (latestMessageTimestamp < msg.timestamp) {
-            if (!this.metadataTimestampsAheadOfRealData.has(msg.messageChainId)) {
-                this.metadataTimestampsAheadOfRealData.set(msg.messageChainId, new Set());
-            }
-            this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).add(msg.timestamp);
-            if (this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).size > 1) {
-                await this.resumeNeighbor(previousNode, msg.messageChainId, this.getLatestMessageTimestamp(msg.messageChainId));
-                this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).forEach((timestamp) => {
-                    this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).delete(timestamp);
-                });
-            }
+        const latestTs = this.getLatestMessageTimestamp(msg.messageChainId);
+        if (latestTs >= msg.timestamp) {
+            return;
+        }
+        const chainId = msg.messageChainId;
+        const cooldownUntil = this.recoveryCooldownUntil.get(chainId);
+        if (cooldownUntil !== undefined && performance.now() < cooldownUntil) {
+            return;
+        }
+        let state = this.recoveryState.get(chainId);
+        if (!state) {
+            state = {
+                timestampsAhead: new Set(),
+                metadataAheadSince: performance.now(),
+                candidates: [],
+                lastAttemptedNode: null,
+                resumeInProgress: false
+            };
+            this.recoveryState.set(chainId, state);
+        }
+        state.timestampsAhead.add(msg.timestamp);
+        const nodeId = toNodeId(previousNode);
+        const isLastAttempted = state.lastAttemptedNode !== null && toNodeId(state.lastAttemptedNode) === nodeId;
+        if (!isLastAttempted && !state.candidates.some((c) => toNodeId(c) === nodeId)) {
+            state.candidates.push(previousNode);
+        }
+        if (state.timestampsAhead.size > 1 && !state.resumeInProgress) {
+            await this.attemptRecovery(chainId, state, latestTs);
+        }
+    }
+    async attemptRecovery(chainId, state, latestTs) {
+        const candidate = state.candidates.shift();
+        if (!candidate) {
+            state.metadataAheadSince = performance.now();
+            return;
+        }
+        state.resumeInProgress = true;
+        state.lastAttemptedNode = candidate;
+        state.candidates = [];
+        state.timestampsAhead.clear();
+        state.metadataAheadSince = performance.now();
+        try {
+            const remote = this.createRemote(candidate);
+            await remote.resumeNeighbor(latestTs, chainId);
+        }
+        catch (_e) {
+            logger$4.debug('Recovery resume failed, will retry with next candidate');
+        }
+        finally {
+            state.resumeInProgress = false;
         }
     }
     createRemote(neighbor) {
@@ -3511,8 +3594,13 @@ class PlumtreeManager extends EventEmitter {
             this.latestMessages.get(messageChainId).shift();
             this.latestMessages.get(messageChainId).push(msg);
         }
-        if (this.metadataTimestampsAheadOfRealData.has(msg.messageId.messageChainId)) {
-            this.metadataTimestampsAheadOfRealData.get(msg.messageId.messageChainId).delete(msg.messageId.timestamp);
+        const state = this.recoveryState.get(messageChainId);
+        if (state) {
+            if (state.lastAttemptedNode) {
+                this.remotePausedNeighbors.delete(toNodeId(state.lastAttemptedNode), messageChainId);
+            }
+            this.recoveryState.delete(messageChainId);
+            this.recoveryCooldownUntil.set(messageChainId, performance.now() + this.recoveryCooldown);
         }
         this.emit('message', msg);
         const neighbors = this.neighbors.getAll().filter((neighbor) => toNodeId(neighbor.getPeerDescriptor()) !== previousNode);
@@ -3530,7 +3618,14 @@ class PlumtreeManager extends EventEmitter {
         return this.localPausedNeighbors.isPaused(toNodeId(node), msgChainId)
             || this.remotePausedNeighbors.isPaused(toNodeId(node), msgChainId);
     }
+    getLocalPausedNeighbors() {
+        return this.localPausedNeighbors;
+    }
+    getRemotePausedNeighbors() {
+        return this.remotePausedNeighbors;
+    }
     stop() {
+        this.abortController.abort();
         this.neighbors.off('nodeRemoved', this.onNeighborRemoved);
     }
 }