@streamr/trackerless-network 103.3.1 → 103.7.0-rc.2

package/dist/exports.cjs

@@ -29,7 +29,7 @@ class ExternalNetworkRpc {
     }
 }
 
-var version = "103.3.1";
+var version = "103.7.0-rc.2";
 
 // @generated message type with reflection information, may provide speed optimized methods
 class Any$Type extends runtime.MessageType {
@@ -1676,6 +1676,18 @@ class PauseNeighborRequest$Type extends runtime.MessageType {
  */
 const PauseNeighborRequest = new PauseNeighborRequest$Type();
 // @generated message type with reflection information, may provide speed optimized methods
+class PauseNeighborResponse$Type extends runtime.MessageType {
+    constructor() {
+        super("PauseNeighborResponse", [
+            { no: 1, name: "accepted", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
+        ]);
+    }
+}
+/**
+ * @generated MessageType for protobuf message PauseNeighborResponse
+ */
+const PauseNeighborResponse = new PauseNeighborResponse$Type();
+// @generated message type with reflection information, may provide speed optimized methods
 class ResumeNeighborRequest$Type extends runtime.MessageType {
     constructor() {
         super("ResumeNeighborRequest", [
@@ -1731,7 +1743,7 @@ const NodeInfoRpc = new runtimeRpc.ServiceType("NodeInfoRpc", [
  * @generated ServiceType for protobuf service PlumtreeRpc
  */
 const PlumtreeRpc = new runtimeRpc.ServiceType("PlumtreeRpc", [
-    { name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: Empty },
+    { name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: PauseNeighborResponse },
     { name: "resumeNeighbor", options: {}, I: ResumeNeighborRequest, O: Empty },
     { name: "sendMetadata", options: {}, I: MessageID, O: Empty }
 ]);
@@ -2392,12 +2404,72 @@ class NeighborUpdateManager {
     }
 }
 
+let enabled = false;
+function setGapDiagnosticsEnabled(val) {
+    enabled = val;
+    globalThis.__trackerlessGapDiagEnabled = val;
+}
+const SUMMARY_INTERVAL_MS = 2000;
+const accumulators = new Map();
+function logGapDiagnosticSampled(layer, opts = {}) {
+    if (!enabled)
+        return;
+    const now = performance.now();
+    const threshold = opts.outlierThresholdMs ?? 30;
+    let acc = accumulators.get(layer);
+    if (acc === undefined) {
+        acc = { count: 0, sumDeltaMs: 0, maxDeltaMs: 0, outlierCount: 0, lastReportMs: now, lastEventMs: now };
+        accumulators.set(layer, acc);
+        return;
+    }
+    const deltaMs = now - acc.lastEventMs;
+    acc.lastEventMs = now;
+    acc.count++;
+    if (deltaMs > acc.maxDeltaMs)
+        acc.maxDeltaMs = deltaMs;
+    acc.sumDeltaMs += deltaMs;
+    if (deltaMs > threshold)
+        acc.outlierCount++;
+    if (deltaMs > threshold) {
+        const payload = {
+            layer,
+            timestampMs: now,
+            deltaMs: +deltaMs.toFixed(2),
+            detail: opts.detail,
+        };
+        // eslint-disable-next-line no-console
+        console.log('[gap-diagnostics]', JSON.stringify(payload));
+    }
+    if (now - acc.lastReportMs >= SUMMARY_INTERVAL_MS) {
+        const summaryDetail = {
+            count: acc.count,
+            meanDeltaMs: acc.count > 0 ? +(acc.sumDeltaMs / acc.count).toFixed(2) : 0,
+            maxDeltaMs: +acc.maxDeltaMs.toFixed(2),
+            outlierCount: acc.outlierCount,
+            periodMs: +(now - acc.lastReportMs).toFixed(1),
+        };
+        const summary = {
+            layer: `${layer}.summary`,
+            timestampMs: now,
+            detail: summaryDetail,
+        };
+        // eslint-disable-next-line no-console
+        console.log('[gap-diagnostics]', JSON.stringify(summary));
+        acc.count = 0;
+        acc.sumDeltaMs = 0;
+        acc.maxDeltaMs = 0;
+        acc.outlierCount = 0;
+        acc.lastReportMs = now;
+    }
+}
+
 class ContentDeliveryRpcLocal {
     options;
     constructor(options) {
         this.options = options;
     }
     async sendStreamMessage(message, context) {
+        logGapDiagnosticSampled('trackerless.rpcLocal.sendStreamMessage');
         const previousNode = context.incomingSourceDescriptor;
         const previousNodeId = dht.toNodeId(previousNode);
         this.options.markForInspection(previousNodeId, message.messageId);
@@ -2758,11 +2830,16 @@ class ContentDeliveryLayerNode extends eventemitter3.EventEmitter {
         this.options.neighborFinder.stop();
         this.options.neighborUpdateManager.stop();
         this.options.inspector.stop();
+        this.duplicateDetectors.clear();
     }
     broadcast(msg, previousNode) {
         if (!previousNode) {
+            logGapDiagnosticSampled('trackerless.cdNode.broadcastOut');
             markAndCheckDuplicate(this.duplicateDetectors, msg.messageId, msg.previousMessageRef);
         }
+        else {
+            logGapDiagnosticSampled('trackerless.cdNode.broadcastIn');
+        }
         this.emit('message', msg);
         const skipBackPropagation = previousNode !== undefined && !this.options.temporaryConnectionRpcLocal.hasNode(previousNode);
         this.options.propagation.feedUnseenMessage(msg, this.getPropagationTargets(msg), skipBackPropagation ? previousNode : null);
@@ -3060,6 +3137,7 @@ class Propagation {
     }
     sendAndAwaitThenMark({ message, source, handledNeighbors }, neighborId) {
         if (!handledNeighbors.has(neighborId) && neighborId !== source) {
+            logGapDiagnosticSampled('trackerless.propagation.sendToNeighbor');
             (async () => {
                 try {
                     await this.sendToNeighbor(neighborId, message);
@@ -3333,14 +3411,17 @@ class PlumtreeRpcLocal {
     async pauseNeighbor(request, context) {
         const sender = dht.toNodeId(context.incomingSourceDescriptor);
         if (this.neighbors.has(sender)) {
-            this.pausedNodes.add(sender, request.messageChainId);
+            const accepted = this.pausedNodes.add(sender, request.messageChainId);
+            return { accepted };
         }
-        return Empty;
+        return { accepted: false };
     }
     async resumeNeighbor(request, context) {
         const sender = context.incomingSourceDescriptor;
-        this.pausedNodes.delete(dht.toNodeId(sender), request.messageChainId);
-        await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
+        if (this.neighbors.has(dht.toNodeId(sender))) {
+            this.pausedNodes.delete(dht.toNodeId(sender), request.messageChainId);
+            await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
+        }
         return Empty;
     }
 }
@@ -3353,10 +3434,9 @@ class PlumtreeRpcRemote extends dht.RpcRemote {
         await this.getClient().sendMetadata(msg, options);
     }
     async pauseNeighbor(messageChainId) {
-        const options = this.formDhtRpcOptions({
-            notification: true
-        });
-        await this.getClient().pauseNeighbor({ messageChainId }, options);
+        const options = this.formDhtRpcOptions();
+        const response = await this.getClient().pauseNeighbor({ messageChainId }, options);
+        return response.accepted;
     }
     async resumeNeighbor(fromTimestamp, messageChainId) {
         const options = this.formDhtRpcOptions({
@@ -3378,9 +3458,10 @@ class PausedNeighbors {
             this.pausedNeighbors.set(msgChainId, new Set());
         }
         if (this.pausedNeighbors.get(msgChainId).size >= this.limit) {
-            return;
+            return false;
         }
         this.pausedNeighbors.get(msgChainId).add(node);
+        return true;
     }
     delete(node, msgChainId) {
         this.pausedNeighbors.get(msgChainId)?.delete(node);
@@ -3413,19 +3494,24 @@ class PausedNeighbors {
 }
 
 const MAX_PAUSED_NEIGHBORS_DEFAULT = 3;
+const DEFAULT_RECOVERY_TIMEOUT = 500;
+const DEFAULT_RECOVERY_CHECK_INTERVAL = 200;
+const DEFAULT_RECOVERY_COOLDOWN = 2500;
 const logger$4 = new utils.Logger('PlumtreeManager');
 class PlumtreeManager extends eventemitter3.EventEmitter {
     neighbors;
     localPeerDescriptor;
-    // We have paused sending real data to these neighbrs and only send metadata
     localPausedNeighbors;
-    // We have asked these nodes to pause sending real data to us, used to limit sending of pausing and resuming requests
     remotePausedNeighbors;
     rpcLocal;
     latestMessages = new Map();
     rpcCommunicator;
-    metadataTimestampsAheadOfRealData = new Map();
     maxPausedNeighbors;
+    recoveryState = new Map();
+    recoveryCooldownUntil = new Map();
+    recoveryTimeout;
+    recoveryCooldown;
+    abortController = new AbortController();
     constructor(options) {
         super();
         this.neighbors = options.neighbors;
@@ -3433,12 +3519,22 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
         this.localPeerDescriptor = options.localPeerDescriptor;
         this.localPausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
         this.remotePausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
+        this.recoveryTimeout = options.recoveryTimeout ?? DEFAULT_RECOVERY_TIMEOUT;
+        this.recoveryCooldown = options.recoveryCooldown ?? DEFAULT_RECOVERY_COOLDOWN;
         this.rpcLocal = new PlumtreeRpcLocal(this.neighbors, this.localPausedNeighbors, (metadata, previousNode) => this.onMetadata(metadata, previousNode), (fromTimestamp, msgChainId, remotePeerDescriptor) => this.sendBuffer(fromTimestamp, msgChainId, remotePeerDescriptor));
-        this.neighbors.on('nodeRemoved', (nodeId) => this.onNeighborRemoved(nodeId));
+        this.neighbors.on('nodeRemoved', this.onNeighborRemoved);
         this.rpcCommunicator = options.rpcCommunicator;
         this.rpcCommunicator.registerRpcNotification(MessageID, 'sendMetadata', (msg, context) => this.rpcLocal.sendMetadata(msg, context));
-        this.rpcCommunicator.registerRpcNotification(PauseNeighborRequest, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
+        this.rpcCommunicator.registerRpcMethod(PauseNeighborRequest, PauseNeighborResponse, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
         this.rpcCommunicator.registerRpcNotification(ResumeNeighborRequest, 'resumeNeighbor', (msg, context) => this.rpcLocal.resumeNeighbor(msg, context));
+        utils.setAbortableInterval(() => {
+            const now = performance.now();
+            for (const [chainId, state] of this.recoveryState) {
+                if (now - state.metadataAheadSince >= this.recoveryTimeout && !state.resumeInProgress) {
+                    this.attemptRecovery(chainId, state, this.getLatestMessageTimestamp(chainId));
+                }
+            }
+        }, options.recoveryCheckInterval ?? DEFAULT_RECOVERY_CHECK_INTERVAL, this.abortController.signal);
     }
     async pauseNeighbor(node, msgChainId) {
         if (this.neighbors.has(dht.toNodeId(node))
@@ -3446,8 +3542,16 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
             && this.remotePausedNeighbors.size(msgChainId) < this.maxPausedNeighbors) {
             logger$4.debug(`Pausing neighbor ${dht.toNodeId(node)}`);
             this.remotePausedNeighbors.add(dht.toNodeId(node), msgChainId);
-            const remote = this.createRemote(node);
-            await remote.pauseNeighbor(msgChainId);
+            try {
+                const remote = this.createRemote(node);
+                const accepted = await remote.pauseNeighbor(msgChainId);
+                if (!accepted) {
+                    this.remotePausedNeighbors.delete(dht.toNodeId(node), msgChainId);
+                }
+            }
+            catch (_e) {
+                this.remotePausedNeighbors.delete(dht.toNodeId(node), msgChainId);
+            }
         }
     }
     async resumeNeighbor(node, msgChainId, fromTimestamp) {
@@ -3458,9 +3562,15 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
             await remote.resumeNeighbor(fromTimestamp, msgChainId);
         }
     }
-    onNeighborRemoved(nodeId) {
+    onNeighborRemoved = (nodeId) => {
         this.localPausedNeighbors.deleteAll(nodeId);
         this.remotePausedNeighbors.deleteAll(nodeId);
+        for (const [_chainId, state] of this.recoveryState) {
+            state.candidates = state.candidates.filter((c) => dht.toNodeId(c) !== nodeId);
+            if (state.lastAttemptedNode !== null && dht.toNodeId(state.lastAttemptedNode) === nodeId) {
+                state.lastAttemptedNode = null;
+            }
+        }
         if (this.neighbors.size() > 0) {
             this.remotePausedNeighbors.forEach((pausedNeighbors, msgChainId) => {
                 if (pausedNeighbors.size >= this.neighbors.size()) {
@@ -3470,7 +3580,7 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
                 }
             });
         }
-    }
+    };
     getLatestMessageTimestamp(msgChainId) {
         if (!this.latestMessages.has(msgChainId) || this.latestMessages.get(msgChainId).length === 0) {
             return 0;
@@ -3480,22 +3590,61 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
     async sendBuffer(fromTimestamp, msgChainId, neighbor) {
         const remote = new ContentDeliveryRpcRemote(this.localPeerDescriptor, neighbor, this.rpcCommunicator, ContentDeliveryRpcClient);
         const messages = this.latestMessages.get(msgChainId)?.filter((msg) => msg.messageId.timestamp > fromTimestamp) ?? [];
-        await Promise.all(messages.map((msg) => remote.sendStreamMessage(msg)));
+        for (const msg of messages) {
+            await remote.sendStreamMessage(msg);
+        }
     }
     async onMetadata(msg, previousNode) {
-        // If we receive newer metadata than messages in the buffer, resume the sending neighbor
-        const latestMessageTimestamp = this.getLatestMessageTimestamp(msg.messageChainId);
-        if (latestMessageTimestamp < msg.timestamp) {
-            if (!this.metadataTimestampsAheadOfRealData.has(msg.messageChainId)) {
-                this.metadataTimestampsAheadOfRealData.set(msg.messageChainId, new Set());
-            }
-            this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).add(msg.timestamp);
-            if (this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).size > 1) {
-                await this.resumeNeighbor(previousNode, msg.messageChainId, this.getLatestMessageTimestamp(msg.messageChainId));
-                this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).forEach((timestamp) => {
-                    this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).delete(timestamp);
-                });
-            }
+        const latestTs = this.getLatestMessageTimestamp(msg.messageChainId);
+        if (latestTs >= msg.timestamp) {
+            return;
+        }
+        const chainId = msg.messageChainId;
+        const cooldownUntil = this.recoveryCooldownUntil.get(chainId);
+        if (cooldownUntil !== undefined && performance.now() < cooldownUntil) {
+            return;
+        }
+        let state = this.recoveryState.get(chainId);
+        if (!state) {
+            state = {
+                timestampsAhead: new Set(),
+                metadataAheadSince: performance.now(),
+                candidates: [],
+                lastAttemptedNode: null,
+                resumeInProgress: false
+            };
+            this.recoveryState.set(chainId, state);
+        }
+        state.timestampsAhead.add(msg.timestamp);
+        const nodeId = dht.toNodeId(previousNode);
+        const isLastAttempted = state.lastAttemptedNode !== null && dht.toNodeId(state.lastAttemptedNode) === nodeId;
+        if (!isLastAttempted && !state.candidates.some((c) => dht.toNodeId(c) === nodeId)) {
+            state.candidates.push(previousNode);
+        }
+        if (state.timestampsAhead.size > 1 && !state.resumeInProgress) {
+            await this.attemptRecovery(chainId, state, latestTs);
+        }
+    }
+    async attemptRecovery(chainId, state, latestTs) {
+        const candidate = state.candidates.shift();
+        if (!candidate) {
+            state.metadataAheadSince = performance.now();
+            return;
+        }
+        state.resumeInProgress = true;
+        state.lastAttemptedNode = candidate;
+        state.candidates = [];
+        state.timestampsAhead.clear();
+        state.metadataAheadSince = performance.now();
+        try {
+            const remote = this.createRemote(candidate);
+            await remote.resumeNeighbor(latestTs, chainId);
+        }
+        catch (_e) {
+            logger$4.debug('Recovery resume failed, will retry with next candidate');
+        }
+        finally {
+            state.resumeInProgress = false;
         }
     }
     createRemote(neighbor) {
@@ -3513,8 +3662,13 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
             this.latestMessages.get(messageChainId).shift();
             this.latestMessages.get(messageChainId).push(msg);
         }
-        if (this.metadataTimestampsAheadOfRealData.has(msg.messageId.messageChainId)) {
-            this.metadataTimestampsAheadOfRealData.get(msg.messageId.messageChainId).delete(msg.messageId.timestamp);
+        const state = this.recoveryState.get(messageChainId);
+        if (state) {
+            if (state.lastAttemptedNode) {
+                this.remotePausedNeighbors.delete(dht.toNodeId(state.lastAttemptedNode), messageChainId);
+            }
+            this.recoveryState.delete(messageChainId);
+            this.recoveryCooldownUntil.set(messageChainId, performance.now() + this.recoveryCooldown);
         }
         this.emit('message', msg);
         const neighbors = this.neighbors.getAll().filter((neighbor) => dht.toNodeId(neighbor.getPeerDescriptor()) !== previousNode);
@@ -3532,8 +3686,18 @@ class PlumtreeManager extends eventemitter3.EventEmitter {
         return this.localPausedNeighbors.isPaused(dht.toNodeId(node), msgChainId)
             || this.remotePausedNeighbors.isPaused(dht.toNodeId(node), msgChainId);
     }
+    getLocalPausedNeighbors() {
+        return this.localPausedNeighbors;
+    }
+    getRemotePausedNeighbors() {
+        return this.remotePausedNeighbors;
+    }
     stop() {
+        this.abortController.abort();
         this.neighbors.off('nodeRemoved', this.onNeighborRemoved);
+        this.latestMessages.clear();
+        this.recoveryState.clear();
+        this.recoveryCooldownUntil.clear();
     }
 }
 
@@ -4485,5 +4649,7 @@ exports.NetworkNode = NetworkNode;
 exports.NetworkStack = NetworkStack;
 exports.StreamMessage = StreamMessage;
 exports.createNetworkNode = createNetworkNode;
+exports.logGapDiagnosticSampled = logGapDiagnosticSampled;
+exports.setTrackerlessGapDiagnosticsEnabled = setGapDiagnosticsEnabled;
 exports.streamPartIdToDataKey = streamPartIdToDataKey;
 //# sourceMappingURL=exports.cjs.map