@streamr/trackerless-network 103.3.1 → 103.7.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/exports.d.ts CHANGED
@@ -1084,11 +1084,26 @@ type ContentDeliveryLayerNeighborInfo = MarkRequired<ContentDeliveryLayerNeighbo
1084
1084
  type StreamPartitionInfo = ChangeFieldType<Required<StreamPartitionInfo$1>, 'contentDeliveryLayerNeighbors', ContentDeliveryLayerNeighborInfo[]>;
1085
1085
  type NodeInfo = ChangeFieldType<Required<NodeInfoResponse>, 'streamPartitions', StreamPartitionInfo[]>;
1086
1086
 
1087
+ declare class PausedNeighbors {
1088
+ private readonly pausedNeighbors;
1089
+ private readonly limit;
1090
+ constructor(limit: number);
1091
+ add(node: DhtAddress, msgChainId: string): boolean;
1092
+ delete(node: DhtAddress, msgChainId: string): void;
1093
+ deleteAll(node: DhtAddress): void;
1094
+ isPaused(node: DhtAddress, msgChainId: string): boolean;
1095
+ forEach(fn: (neighbors: Set<DhtAddress>, msgChainId: string) => void): void;
1096
+ size(msgChainId: string): number;
1097
+ }
1098
+
1087
1099
  interface Options {
1088
1100
  neighbors: NodeList;
1089
1101
  localPeerDescriptor: PeerDescriptor$1;
1090
1102
  rpcCommunicator: ListeningRpcCommunicator;
1091
1103
  maxPausedNeighbors?: number;
1104
+ recoveryTimeout?: number;
1105
+ recoveryCheckInterval?: number;
1106
+ recoveryCooldown?: number;
1092
1107
  }
1093
1108
  interface Events$3 {
1094
1109
  message: (msg: StreamMessage) => void;
@@ -1101,8 +1116,12 @@ declare class PlumtreeManager extends EventEmitter<Events$3> {
1101
1116
  private readonly rpcLocal;
1102
1117
  private readonly latestMessages;
1103
1118
  private readonly rpcCommunicator;
1104
- private readonly metadataTimestampsAheadOfRealData;
1105
1119
  private readonly maxPausedNeighbors;
1120
+ private readonly recoveryState;
1121
+ private readonly recoveryCooldownUntil;
1122
+ private readonly recoveryTimeout;
1123
+ private readonly recoveryCooldown;
1124
+ private readonly abortController;
1106
1125
  constructor(options: Options);
1107
1126
  pauseNeighbor(node: PeerDescriptor$1, msgChainId: string): Promise<void>;
1108
1127
  resumeNeighbor(node: PeerDescriptor$1, msgChainId: string, fromTimestamp: number): Promise<void>;
@@ -1110,9 +1129,12 @@ declare class PlumtreeManager extends EventEmitter<Events$3> {
1110
1129
  getLatestMessageTimestamp(msgChainId: string): number;
1111
1130
  private sendBuffer;
1112
1131
  private onMetadata;
1132
+ private attemptRecovery;
1113
1133
  private createRemote;
1114
1134
  broadcast(msg: StreamMessage, previousNode: DhtAddress): void;
1115
1135
  isNeighborPaused(node: PeerDescriptor$1, msgChainId: string): boolean;
1136
+ getLocalPausedNeighbors(): PausedNeighbors;
1137
+ getRemotePausedNeighbors(): PausedNeighbors;
1116
1138
  stop(): void;
1117
1139
  }
1118
1140
 
@@ -1387,5 +1409,11 @@ declare class NetworkNode {
1387
1409
  createExternalRpcClient<T extends ExternalRpcClient>(clientClass: ExternalRpcClientClass<T>): ProtoRpcClient<T>;
1388
1410
  }
1389
1411
 
1390
- export { AsymmetricEncryptionType, ContentType, ControlLayerInfo, EncryptedGroupKey, EncryptionType, GroupKeyRequest, GroupKeyResponse, MessageID, MessageRef, NetworkNode, NetworkStack, ProxyDirection, SignatureType, StreamMessage, createNetworkNode, streamPartIdToDataKey };
1412
+ declare function setGapDiagnosticsEnabled(val: boolean): void;
1413
+ declare function logGapDiagnosticSampled(layer: string, opts?: {
1414
+ detail?: Record<string, unknown>;
1415
+ outlierThresholdMs?: number;
1416
+ }): void;
1417
+
1418
+ export { AsymmetricEncryptionType, ContentType, ControlLayerInfo, EncryptedGroupKey, EncryptionType, GroupKeyRequest, GroupKeyResponse, MessageID, MessageRef, NetworkNode, NetworkStack, ProxyDirection, SignatureType, StreamMessage, createNetworkNode, logGapDiagnosticSampled, setGapDiagnosticsEnabled as setTrackerlessGapDiagnosticsEnabled, streamPartIdToDataKey };
1391
1419
  export type { ContentDeliveryLayerNeighborInfo, ContentDeliveryManagerOptions, ExternalRpcClient, ExternalRpcClientClass, NetworkOptions, NodeInfo, StreamPartDeliveryOptions, StreamPartitionInfo };
package/dist/exports.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { ListeningRpcCommunicator, PeerDescriptor as PeerDescriptor$1, areEqualPeerDescriptors, toNodeId, RpcRemote, toDhtAddress, toDhtAddressRaw, EXISTING_CONNECTION_TIMEOUT, DhtNode } from '@streamr/dht';
2
2
  import { toProtoRpcClient } from '@streamr/proto-rpc';
3
- import { Logger, scheduleAtInterval, wait, setAbortableTimeout, toUserId, addManagedEventListener, waitForEvent, toUserIdRaw, computeSha1, MetricsContext, RateMetric, toStreamPartID, StreamPartIDUtils, until } from '@streamr/utils';
3
+ import { Logger, scheduleAtInterval, wait, setAbortableTimeout, toUserId, addManagedEventListener, waitForEvent, setAbortableInterval, toUserIdRaw, computeSha1, MetricsContext, RateMetric, toStreamPartID, StreamPartIDUtils, until } from '@streamr/utils';
4
4
  import pull from 'lodash/pull';
5
5
  import { EventEmitter } from 'eventemitter3';
6
6
  import sampleSize from 'lodash/sampleSize';
@@ -27,7 +27,7 @@ class ExternalNetworkRpc {
27
27
  }
28
28
  }
29
29
 
30
- var version = "103.3.1";
30
+ var version = "103.7.0-rc.2";
31
31
 
32
32
  // @generated message type with reflection information, may provide speed optimized methods
33
33
  class Any$Type extends MessageType {
@@ -1674,6 +1674,18 @@ class PauseNeighborRequest$Type extends MessageType {
1674
1674
  */
1675
1675
  const PauseNeighborRequest = new PauseNeighborRequest$Type();
1676
1676
  // @generated message type with reflection information, may provide speed optimized methods
1677
+ class PauseNeighborResponse$Type extends MessageType {
1678
+ constructor() {
1679
+ super("PauseNeighborResponse", [
1680
+ { no: 1, name: "accepted", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
1681
+ ]);
1682
+ }
1683
+ }
1684
+ /**
1685
+ * @generated MessageType for protobuf message PauseNeighborResponse
1686
+ */
1687
+ const PauseNeighborResponse = new PauseNeighborResponse$Type();
1688
+ // @generated message type with reflection information, may provide speed optimized methods
1677
1689
  class ResumeNeighborRequest$Type extends MessageType {
1678
1690
  constructor() {
1679
1691
  super("ResumeNeighborRequest", [
@@ -1729,7 +1741,7 @@ const NodeInfoRpc = new ServiceType("NodeInfoRpc", [
1729
1741
  * @generated ServiceType for protobuf service PlumtreeRpc
1730
1742
  */
1731
1743
  const PlumtreeRpc = new ServiceType("PlumtreeRpc", [
1732
- { name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: Empty },
1744
+ { name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: PauseNeighborResponse },
1733
1745
  { name: "resumeNeighbor", options: {}, I: ResumeNeighborRequest, O: Empty },
1734
1746
  { name: "sendMetadata", options: {}, I: MessageID, O: Empty }
1735
1747
  ]);
@@ -2390,12 +2402,72 @@ class NeighborUpdateManager {
2390
2402
  }
2391
2403
  }
2392
2404
 
2405
+ let enabled = false;
2406
+ function setGapDiagnosticsEnabled(val) {
2407
+ enabled = val;
2408
+ globalThis.__trackerlessGapDiagEnabled = val;
2409
+ }
2410
+ const SUMMARY_INTERVAL_MS = 2000;
2411
+ const accumulators = new Map();
2412
+ function logGapDiagnosticSampled(layer, opts = {}) {
2413
+ if (!enabled)
2414
+ return;
2415
+ const now = performance.now();
2416
+ const threshold = opts.outlierThresholdMs ?? 30;
2417
+ let acc = accumulators.get(layer);
2418
+ if (acc === undefined) {
2419
+ acc = { count: 0, sumDeltaMs: 0, maxDeltaMs: 0, outlierCount: 0, lastReportMs: now, lastEventMs: now };
2420
+ accumulators.set(layer, acc);
2421
+ return;
2422
+ }
2423
+ const deltaMs = now - acc.lastEventMs;
2424
+ acc.lastEventMs = now;
2425
+ acc.count++;
2426
+ if (deltaMs > acc.maxDeltaMs)
2427
+ acc.maxDeltaMs = deltaMs;
2428
+ acc.sumDeltaMs += deltaMs;
2429
+ if (deltaMs > threshold)
2430
+ acc.outlierCount++;
2431
+ if (deltaMs > threshold) {
2432
+ const payload = {
2433
+ layer,
2434
+ timestampMs: now,
2435
+ deltaMs: +deltaMs.toFixed(2),
2436
+ detail: opts.detail,
2437
+ };
2438
+ // eslint-disable-next-line no-console
2439
+ console.log('[gap-diagnostics]', JSON.stringify(payload));
2440
+ }
2441
+ if (now - acc.lastReportMs >= SUMMARY_INTERVAL_MS) {
2442
+ const summaryDetail = {
2443
+ count: acc.count,
2444
+ meanDeltaMs: acc.count > 0 ? +(acc.sumDeltaMs / acc.count).toFixed(2) : 0,
2445
+ maxDeltaMs: +acc.maxDeltaMs.toFixed(2),
2446
+ outlierCount: acc.outlierCount,
2447
+ periodMs: +(now - acc.lastReportMs).toFixed(1),
2448
+ };
2449
+ const summary = {
2450
+ layer: `${layer}.summary`,
2451
+ timestampMs: now,
2452
+ detail: summaryDetail,
2453
+ };
2454
+ // eslint-disable-next-line no-console
2455
+ console.log('[gap-diagnostics]', JSON.stringify(summary));
2456
+ acc.count = 0;
2457
+ acc.sumDeltaMs = 0;
2458
+ acc.maxDeltaMs = 0;
2459
+ acc.outlierCount = 0;
2460
+ acc.lastReportMs = now;
2461
+ }
2462
+ }
2463
+
2393
2464
  class ContentDeliveryRpcLocal {
2394
2465
  options;
2395
2466
  constructor(options) {
2396
2467
  this.options = options;
2397
2468
  }
2398
2469
  async sendStreamMessage(message, context) {
2470
+ logGapDiagnosticSampled('trackerless.rpcLocal.sendStreamMessage');
2399
2471
  const previousNode = context.incomingSourceDescriptor;
2400
2472
  const previousNodeId = toNodeId(previousNode);
2401
2473
  this.options.markForInspection(previousNodeId, message.messageId);
@@ -2756,11 +2828,16 @@ class ContentDeliveryLayerNode extends EventEmitter {
2756
2828
  this.options.neighborFinder.stop();
2757
2829
  this.options.neighborUpdateManager.stop();
2758
2830
  this.options.inspector.stop();
2831
+ this.duplicateDetectors.clear();
2759
2832
  }
2760
2833
  broadcast(msg, previousNode) {
2761
2834
  if (!previousNode) {
2835
+ logGapDiagnosticSampled('trackerless.cdNode.broadcastOut');
2762
2836
  markAndCheckDuplicate(this.duplicateDetectors, msg.messageId, msg.previousMessageRef);
2763
2837
  }
2838
+ else {
2839
+ logGapDiagnosticSampled('trackerless.cdNode.broadcastIn');
2840
+ }
2764
2841
  this.emit('message', msg);
2765
2842
  const skipBackPropagation = previousNode !== undefined && !this.options.temporaryConnectionRpcLocal.hasNode(previousNode);
2766
2843
  this.options.propagation.feedUnseenMessage(msg, this.getPropagationTargets(msg), skipBackPropagation ? previousNode : null);
@@ -3058,6 +3135,7 @@ class Propagation {
3058
3135
  }
3059
3136
  sendAndAwaitThenMark({ message, source, handledNeighbors }, neighborId) {
3060
3137
  if (!handledNeighbors.has(neighborId) && neighborId !== source) {
3138
+ logGapDiagnosticSampled('trackerless.propagation.sendToNeighbor');
3061
3139
  (async () => {
3062
3140
  try {
3063
3141
  await this.sendToNeighbor(neighborId, message);
@@ -3331,14 +3409,17 @@ class PlumtreeRpcLocal {
3331
3409
  async pauseNeighbor(request, context) {
3332
3410
  const sender = toNodeId(context.incomingSourceDescriptor);
3333
3411
  if (this.neighbors.has(sender)) {
3334
- this.pausedNodes.add(sender, request.messageChainId);
3412
+ const accepted = this.pausedNodes.add(sender, request.messageChainId);
3413
+ return { accepted };
3335
3414
  }
3336
- return Empty;
3415
+ return { accepted: false };
3337
3416
  }
3338
3417
  async resumeNeighbor(request, context) {
3339
3418
  const sender = context.incomingSourceDescriptor;
3340
- this.pausedNodes.delete(toNodeId(sender), request.messageChainId);
3341
- await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
3419
+ if (this.neighbors.has(toNodeId(sender))) {
3420
+ this.pausedNodes.delete(toNodeId(sender), request.messageChainId);
3421
+ await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
3422
+ }
3342
3423
  return Empty;
3343
3424
  }
3344
3425
  }
@@ -3351,10 +3432,9 @@ class PlumtreeRpcRemote extends RpcRemote {
3351
3432
  await this.getClient().sendMetadata(msg, options);
3352
3433
  }
3353
3434
  async pauseNeighbor(messageChainId) {
3354
- const options = this.formDhtRpcOptions({
3355
- notification: true
3356
- });
3357
- await this.getClient().pauseNeighbor({ messageChainId }, options);
3435
+ const options = this.formDhtRpcOptions();
3436
+ const response = await this.getClient().pauseNeighbor({ messageChainId }, options);
3437
+ return response.accepted;
3358
3438
  }
3359
3439
  async resumeNeighbor(fromTimestamp, messageChainId) {
3360
3440
  const options = this.formDhtRpcOptions({
@@ -3376,9 +3456,10 @@ class PausedNeighbors {
3376
3456
  this.pausedNeighbors.set(msgChainId, new Set());
3377
3457
  }
3378
3458
  if (this.pausedNeighbors.get(msgChainId).size >= this.limit) {
3379
- return;
3459
+ return false;
3380
3460
  }
3381
3461
  this.pausedNeighbors.get(msgChainId).add(node);
3462
+ return true;
3382
3463
  }
3383
3464
  delete(node, msgChainId) {
3384
3465
  this.pausedNeighbors.get(msgChainId)?.delete(node);
@@ -3411,19 +3492,24 @@ class PausedNeighbors {
3411
3492
  }
3412
3493
 
3413
3494
  const MAX_PAUSED_NEIGHBORS_DEFAULT = 3;
3495
+ const DEFAULT_RECOVERY_TIMEOUT = 500;
3496
+ const DEFAULT_RECOVERY_CHECK_INTERVAL = 200;
3497
+ const DEFAULT_RECOVERY_COOLDOWN = 2500;
3414
3498
  const logger$4 = new Logger('PlumtreeManager');
3415
3499
  class PlumtreeManager extends EventEmitter {
3416
3500
  neighbors;
3417
3501
  localPeerDescriptor;
3418
- // We have paused sending real data to these neighbrs and only send metadata
3419
3502
  localPausedNeighbors;
3420
- // We have asked these nodes to pause sending real data to us, used to limit sending of pausing and resuming requests
3421
3503
  remotePausedNeighbors;
3422
3504
  rpcLocal;
3423
3505
  latestMessages = new Map();
3424
3506
  rpcCommunicator;
3425
- metadataTimestampsAheadOfRealData = new Map();
3426
3507
  maxPausedNeighbors;
3508
+ recoveryState = new Map();
3509
+ recoveryCooldownUntil = new Map();
3510
+ recoveryTimeout;
3511
+ recoveryCooldown;
3512
+ abortController = new AbortController();
3427
3513
  constructor(options) {
3428
3514
  super();
3429
3515
  this.neighbors = options.neighbors;
@@ -3431,12 +3517,22 @@ class PlumtreeManager extends EventEmitter {
3431
3517
  this.localPeerDescriptor = options.localPeerDescriptor;
3432
3518
  this.localPausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
3433
3519
  this.remotePausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
3520
+ this.recoveryTimeout = options.recoveryTimeout ?? DEFAULT_RECOVERY_TIMEOUT;
3521
+ this.recoveryCooldown = options.recoveryCooldown ?? DEFAULT_RECOVERY_COOLDOWN;
3434
3522
  this.rpcLocal = new PlumtreeRpcLocal(this.neighbors, this.localPausedNeighbors, (metadata, previousNode) => this.onMetadata(metadata, previousNode), (fromTimestamp, msgChainId, remotePeerDescriptor) => this.sendBuffer(fromTimestamp, msgChainId, remotePeerDescriptor));
3435
- this.neighbors.on('nodeRemoved', (nodeId) => this.onNeighborRemoved(nodeId));
3523
+ this.neighbors.on('nodeRemoved', this.onNeighborRemoved);
3436
3524
  this.rpcCommunicator = options.rpcCommunicator;
3437
3525
  this.rpcCommunicator.registerRpcNotification(MessageID, 'sendMetadata', (msg, context) => this.rpcLocal.sendMetadata(msg, context));
3438
- this.rpcCommunicator.registerRpcNotification(PauseNeighborRequest, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
3526
+ this.rpcCommunicator.registerRpcMethod(PauseNeighborRequest, PauseNeighborResponse, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
3439
3527
  this.rpcCommunicator.registerRpcNotification(ResumeNeighborRequest, 'resumeNeighbor', (msg, context) => this.rpcLocal.resumeNeighbor(msg, context));
3528
+ setAbortableInterval(() => {
3529
+ const now = performance.now();
3530
+ for (const [chainId, state] of this.recoveryState) {
3531
+ if (now - state.metadataAheadSince >= this.recoveryTimeout && !state.resumeInProgress) {
3532
+ this.attemptRecovery(chainId, state, this.getLatestMessageTimestamp(chainId));
3533
+ }
3534
+ }
3535
+ }, options.recoveryCheckInterval ?? DEFAULT_RECOVERY_CHECK_INTERVAL, this.abortController.signal);
3440
3536
  }
3441
3537
  async pauseNeighbor(node, msgChainId) {
3442
3538
  if (this.neighbors.has(toNodeId(node))
@@ -3444,8 +3540,16 @@ class PlumtreeManager extends EventEmitter {
3444
3540
  && this.remotePausedNeighbors.size(msgChainId) < this.maxPausedNeighbors) {
3445
3541
  logger$4.debug(`Pausing neighbor ${toNodeId(node)}`);
3446
3542
  this.remotePausedNeighbors.add(toNodeId(node), msgChainId);
3447
- const remote = this.createRemote(node);
3448
- await remote.pauseNeighbor(msgChainId);
3543
+ try {
3544
+ const remote = this.createRemote(node);
3545
+ const accepted = await remote.pauseNeighbor(msgChainId);
3546
+ if (!accepted) {
3547
+ this.remotePausedNeighbors.delete(toNodeId(node), msgChainId);
3548
+ }
3549
+ }
3550
+ catch (_e) {
3551
+ this.remotePausedNeighbors.delete(toNodeId(node), msgChainId);
3552
+ }
3449
3553
  }
3450
3554
  }
3451
3555
  async resumeNeighbor(node, msgChainId, fromTimestamp) {
@@ -3456,9 +3560,15 @@ class PlumtreeManager extends EventEmitter {
3456
3560
  await remote.resumeNeighbor(fromTimestamp, msgChainId);
3457
3561
  }
3458
3562
  }
3459
- onNeighborRemoved(nodeId) {
3563
+ onNeighborRemoved = (nodeId) => {
3460
3564
  this.localPausedNeighbors.deleteAll(nodeId);
3461
3565
  this.remotePausedNeighbors.deleteAll(nodeId);
3566
+ for (const [_chainId, state] of this.recoveryState) {
3567
+ state.candidates = state.candidates.filter((c) => toNodeId(c) !== nodeId);
3568
+ if (state.lastAttemptedNode !== null && toNodeId(state.lastAttemptedNode) === nodeId) {
3569
+ state.lastAttemptedNode = null;
3570
+ }
3571
+ }
3462
3572
  if (this.neighbors.size() > 0) {
3463
3573
  this.remotePausedNeighbors.forEach((pausedNeighbors, msgChainId) => {
3464
3574
  if (pausedNeighbors.size >= this.neighbors.size()) {
@@ -3468,7 +3578,7 @@ class PlumtreeManager extends EventEmitter {
3468
3578
  }
3469
3579
  });
3470
3580
  }
3471
- }
3581
+ };
3472
3582
  getLatestMessageTimestamp(msgChainId) {
3473
3583
  if (!this.latestMessages.has(msgChainId) || this.latestMessages.get(msgChainId).length === 0) {
3474
3584
  return 0;
@@ -3478,22 +3588,61 @@ class PlumtreeManager extends EventEmitter {
3478
3588
  async sendBuffer(fromTimestamp, msgChainId, neighbor) {
3479
3589
  const remote = new ContentDeliveryRpcRemote(this.localPeerDescriptor, neighbor, this.rpcCommunicator, ContentDeliveryRpcClient);
3480
3590
  const messages = this.latestMessages.get(msgChainId)?.filter((msg) => msg.messageId.timestamp > fromTimestamp) ?? [];
3481
- await Promise.all(messages.map((msg) => remote.sendStreamMessage(msg)));
3591
+ for (const msg of messages) {
3592
+ await remote.sendStreamMessage(msg);
3593
+ }
3482
3594
  }
3483
3595
  async onMetadata(msg, previousNode) {
3484
- // If we receive newer metadata than messages in the buffer, resume the sending neighbor
3485
- const latestMessageTimestamp = this.getLatestMessageTimestamp(msg.messageChainId);
3486
- if (latestMessageTimestamp < msg.timestamp) {
3487
- if (!this.metadataTimestampsAheadOfRealData.has(msg.messageChainId)) {
3488
- this.metadataTimestampsAheadOfRealData.set(msg.messageChainId, new Set());
3489
- }
3490
- this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).add(msg.timestamp);
3491
- if (this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).size > 1) {
3492
- await this.resumeNeighbor(previousNode, msg.messageChainId, this.getLatestMessageTimestamp(msg.messageChainId));
3493
- this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).forEach((timestamp) => {
3494
- this.metadataTimestampsAheadOfRealData.get(msg.messageChainId).delete(timestamp);
3495
- });
3496
- }
3596
+ const latestTs = this.getLatestMessageTimestamp(msg.messageChainId);
3597
+ if (latestTs >= msg.timestamp) {
3598
+ return;
3599
+ }
3600
+ const chainId = msg.messageChainId;
3601
+ const cooldownUntil = this.recoveryCooldownUntil.get(chainId);
3602
+ if (cooldownUntil !== undefined && performance.now() < cooldownUntil) {
3603
+ return;
3604
+ }
3605
+ let state = this.recoveryState.get(chainId);
3606
+ if (!state) {
3607
+ state = {
3608
+ timestampsAhead: new Set(),
3609
+ metadataAheadSince: performance.now(),
3610
+ candidates: [],
3611
+ lastAttemptedNode: null,
3612
+ resumeInProgress: false
3613
+ };
3614
+ this.recoveryState.set(chainId, state);
3615
+ }
3616
+ state.timestampsAhead.add(msg.timestamp);
3617
+ const nodeId = toNodeId(previousNode);
3618
+ const isLastAttempted = state.lastAttemptedNode !== null && toNodeId(state.lastAttemptedNode) === nodeId;
3619
+ if (!isLastAttempted && !state.candidates.some((c) => toNodeId(c) === nodeId)) {
3620
+ state.candidates.push(previousNode);
3621
+ }
3622
+ if (state.timestampsAhead.size > 1 && !state.resumeInProgress) {
3623
+ await this.attemptRecovery(chainId, state, latestTs);
3624
+ }
3625
+ }
3626
+ async attemptRecovery(chainId, state, latestTs) {
3627
+ const candidate = state.candidates.shift();
3628
+ if (!candidate) {
3629
+ state.metadataAheadSince = performance.now();
3630
+ return;
3631
+ }
3632
+ state.resumeInProgress = true;
3633
+ state.lastAttemptedNode = candidate;
3634
+ state.candidates = [];
3635
+ state.timestampsAhead.clear();
3636
+ state.metadataAheadSince = performance.now();
3637
+ try {
3638
+ const remote = this.createRemote(candidate);
3639
+ await remote.resumeNeighbor(latestTs, chainId);
3640
+ }
3641
+ catch (_e) {
3642
+ logger$4.debug('Recovery resume failed, will retry with next candidate');
3643
+ }
3644
+ finally {
3645
+ state.resumeInProgress = false;
3497
3646
  }
3498
3647
  }
3499
3648
  createRemote(neighbor) {
@@ -3511,8 +3660,13 @@ class PlumtreeManager extends EventEmitter {
3511
3660
  this.latestMessages.get(messageChainId).shift();
3512
3661
  this.latestMessages.get(messageChainId).push(msg);
3513
3662
  }
3514
- if (this.metadataTimestampsAheadOfRealData.has(msg.messageId.messageChainId)) {
3515
- this.metadataTimestampsAheadOfRealData.get(msg.messageId.messageChainId).delete(msg.messageId.timestamp);
3663
+ const state = this.recoveryState.get(messageChainId);
3664
+ if (state) {
3665
+ if (state.lastAttemptedNode) {
3666
+ this.remotePausedNeighbors.delete(toNodeId(state.lastAttemptedNode), messageChainId);
3667
+ }
3668
+ this.recoveryState.delete(messageChainId);
3669
+ this.recoveryCooldownUntil.set(messageChainId, performance.now() + this.recoveryCooldown);
3516
3670
  }
3517
3671
  this.emit('message', msg);
3518
3672
  const neighbors = this.neighbors.getAll().filter((neighbor) => toNodeId(neighbor.getPeerDescriptor()) !== previousNode);
@@ -3530,8 +3684,18 @@ class PlumtreeManager extends EventEmitter {
3530
3684
  return this.localPausedNeighbors.isPaused(toNodeId(node), msgChainId)
3531
3685
  || this.remotePausedNeighbors.isPaused(toNodeId(node), msgChainId);
3532
3686
  }
3687
+ getLocalPausedNeighbors() {
3688
+ return this.localPausedNeighbors;
3689
+ }
3690
+ getRemotePausedNeighbors() {
3691
+ return this.remotePausedNeighbors;
3692
+ }
3533
3693
  stop() {
3694
+ this.abortController.abort();
3534
3695
  this.neighbors.off('nodeRemoved', this.onNeighborRemoved);
3696
+ this.latestMessages.clear();
3697
+ this.recoveryState.clear();
3698
+ this.recoveryCooldownUntil.clear();
3535
3699
  }
3536
3700
  }
3537
3701
 
@@ -4473,5 +4637,5 @@ class NetworkNode {
4473
4637
  }
4474
4638
  }
4475
4639
 
4476
- export { AsymmetricEncryptionType, ContentType, ControlLayerInfo, EncryptedGroupKey, EncryptionType, GroupKeyRequest, GroupKeyResponse, MessageID, MessageRef, NetworkNode, NetworkStack, ProxyDirection, SignatureType, StreamMessage, createNetworkNode, streamPartIdToDataKey };
4640
+ export { AsymmetricEncryptionType, ContentType, ControlLayerInfo, EncryptedGroupKey, EncryptionType, GroupKeyRequest, GroupKeyResponse, MessageID, MessageRef, NetworkNode, NetworkStack, ProxyDirection, SignatureType, StreamMessage, createNetworkNode, logGapDiagnosticSampled, setGapDiagnosticsEnabled as setTrackerlessGapDiagnosticsEnabled, streamPartIdToDataKey };
4477
4641
  //# sourceMappingURL=exports.js.map