@streamr/trackerless-network 103.3.1 → 103.7.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/exports.cjs +202 -36
- package/dist/exports.cjs.map +1 -1
- package/dist/exports.d.ts +30 -2
- package/dist/exports.js +202 -38
- package/dist/exports.js.map +1 -1
- package/package.json +5 -5
package/dist/exports.d.ts
CHANGED
|
@@ -1084,11 +1084,26 @@ type ContentDeliveryLayerNeighborInfo = MarkRequired<ContentDeliveryLayerNeighbo
|
|
|
1084
1084
|
type StreamPartitionInfo = ChangeFieldType<Required<StreamPartitionInfo$1>, 'contentDeliveryLayerNeighbors', ContentDeliveryLayerNeighborInfo[]>;
|
|
1085
1085
|
type NodeInfo = ChangeFieldType<Required<NodeInfoResponse>, 'streamPartitions', StreamPartitionInfo[]>;
|
|
1086
1086
|
|
|
1087
|
+
declare class PausedNeighbors {
|
|
1088
|
+
private readonly pausedNeighbors;
|
|
1089
|
+
private readonly limit;
|
|
1090
|
+
constructor(limit: number);
|
|
1091
|
+
add(node: DhtAddress, msgChainId: string): boolean;
|
|
1092
|
+
delete(node: DhtAddress, msgChainId: string): void;
|
|
1093
|
+
deleteAll(node: DhtAddress): void;
|
|
1094
|
+
isPaused(node: DhtAddress, msgChainId: string): boolean;
|
|
1095
|
+
forEach(fn: (neighbors: Set<DhtAddress>, msgChainId: string) => void): void;
|
|
1096
|
+
size(msgChainId: string): number;
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1087
1099
|
interface Options {
|
|
1088
1100
|
neighbors: NodeList;
|
|
1089
1101
|
localPeerDescriptor: PeerDescriptor$1;
|
|
1090
1102
|
rpcCommunicator: ListeningRpcCommunicator;
|
|
1091
1103
|
maxPausedNeighbors?: number;
|
|
1104
|
+
recoveryTimeout?: number;
|
|
1105
|
+
recoveryCheckInterval?: number;
|
|
1106
|
+
recoveryCooldown?: number;
|
|
1092
1107
|
}
|
|
1093
1108
|
interface Events$3 {
|
|
1094
1109
|
message: (msg: StreamMessage) => void;
|
|
@@ -1101,8 +1116,12 @@ declare class PlumtreeManager extends EventEmitter<Events$3> {
|
|
|
1101
1116
|
private readonly rpcLocal;
|
|
1102
1117
|
private readonly latestMessages;
|
|
1103
1118
|
private readonly rpcCommunicator;
|
|
1104
|
-
private readonly metadataTimestampsAheadOfRealData;
|
|
1105
1119
|
private readonly maxPausedNeighbors;
|
|
1120
|
+
private readonly recoveryState;
|
|
1121
|
+
private readonly recoveryCooldownUntil;
|
|
1122
|
+
private readonly recoveryTimeout;
|
|
1123
|
+
private readonly recoveryCooldown;
|
|
1124
|
+
private readonly abortController;
|
|
1106
1125
|
constructor(options: Options);
|
|
1107
1126
|
pauseNeighbor(node: PeerDescriptor$1, msgChainId: string): Promise<void>;
|
|
1108
1127
|
resumeNeighbor(node: PeerDescriptor$1, msgChainId: string, fromTimestamp: number): Promise<void>;
|
|
@@ -1110,9 +1129,12 @@ declare class PlumtreeManager extends EventEmitter<Events$3> {
|
|
|
1110
1129
|
getLatestMessageTimestamp(msgChainId: string): number;
|
|
1111
1130
|
private sendBuffer;
|
|
1112
1131
|
private onMetadata;
|
|
1132
|
+
private attemptRecovery;
|
|
1113
1133
|
private createRemote;
|
|
1114
1134
|
broadcast(msg: StreamMessage, previousNode: DhtAddress): void;
|
|
1115
1135
|
isNeighborPaused(node: PeerDescriptor$1, msgChainId: string): boolean;
|
|
1136
|
+
getLocalPausedNeighbors(): PausedNeighbors;
|
|
1137
|
+
getRemotePausedNeighbors(): PausedNeighbors;
|
|
1116
1138
|
stop(): void;
|
|
1117
1139
|
}
|
|
1118
1140
|
|
|
@@ -1387,5 +1409,11 @@ declare class NetworkNode {
|
|
|
1387
1409
|
createExternalRpcClient<T extends ExternalRpcClient>(clientClass: ExternalRpcClientClass<T>): ProtoRpcClient<T>;
|
|
1388
1410
|
}
|
|
1389
1411
|
|
|
1390
|
-
|
|
1412
|
+
declare function setGapDiagnosticsEnabled(val: boolean): void;
|
|
1413
|
+
declare function logGapDiagnosticSampled(layer: string, opts?: {
|
|
1414
|
+
detail?: Record<string, unknown>;
|
|
1415
|
+
outlierThresholdMs?: number;
|
|
1416
|
+
}): void;
|
|
1417
|
+
|
|
1418
|
+
export { AsymmetricEncryptionType, ContentType, ControlLayerInfo, EncryptedGroupKey, EncryptionType, GroupKeyRequest, GroupKeyResponse, MessageID, MessageRef, NetworkNode, NetworkStack, ProxyDirection, SignatureType, StreamMessage, createNetworkNode, logGapDiagnosticSampled, setGapDiagnosticsEnabled as setTrackerlessGapDiagnosticsEnabled, streamPartIdToDataKey };
|
|
1391
1419
|
export type { ContentDeliveryLayerNeighborInfo, ContentDeliveryManagerOptions, ExternalRpcClient, ExternalRpcClientClass, NetworkOptions, NodeInfo, StreamPartDeliveryOptions, StreamPartitionInfo };
|
package/dist/exports.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ListeningRpcCommunicator, PeerDescriptor as PeerDescriptor$1, areEqualPeerDescriptors, toNodeId, RpcRemote, toDhtAddress, toDhtAddressRaw, EXISTING_CONNECTION_TIMEOUT, DhtNode } from '@streamr/dht';
|
|
2
2
|
import { toProtoRpcClient } from '@streamr/proto-rpc';
|
|
3
|
-
import { Logger, scheduleAtInterval, wait, setAbortableTimeout, toUserId, addManagedEventListener, waitForEvent, toUserIdRaw, computeSha1, MetricsContext, RateMetric, toStreamPartID, StreamPartIDUtils, until } from '@streamr/utils';
|
|
3
|
+
import { Logger, scheduleAtInterval, wait, setAbortableTimeout, toUserId, addManagedEventListener, waitForEvent, setAbortableInterval, toUserIdRaw, computeSha1, MetricsContext, RateMetric, toStreamPartID, StreamPartIDUtils, until } from '@streamr/utils';
|
|
4
4
|
import pull from 'lodash/pull';
|
|
5
5
|
import { EventEmitter } from 'eventemitter3';
|
|
6
6
|
import sampleSize from 'lodash/sampleSize';
|
|
@@ -27,7 +27,7 @@ class ExternalNetworkRpc {
|
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
var version = "103.
|
|
30
|
+
var version = "103.7.0-rc.2";
|
|
31
31
|
|
|
32
32
|
// @generated message type with reflection information, may provide speed optimized methods
|
|
33
33
|
class Any$Type extends MessageType {
|
|
@@ -1674,6 +1674,18 @@ class PauseNeighborRequest$Type extends MessageType {
|
|
|
1674
1674
|
*/
|
|
1675
1675
|
const PauseNeighborRequest = new PauseNeighborRequest$Type();
|
|
1676
1676
|
// @generated message type with reflection information, may provide speed optimized methods
|
|
1677
|
+
class PauseNeighborResponse$Type extends MessageType {
|
|
1678
|
+
constructor() {
|
|
1679
|
+
super("PauseNeighborResponse", [
|
|
1680
|
+
{ no: 1, name: "accepted", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
|
|
1681
|
+
]);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
/**
|
|
1685
|
+
* @generated MessageType for protobuf message PauseNeighborResponse
|
|
1686
|
+
*/
|
|
1687
|
+
const PauseNeighborResponse = new PauseNeighborResponse$Type();
|
|
1688
|
+
// @generated message type with reflection information, may provide speed optimized methods
|
|
1677
1689
|
class ResumeNeighborRequest$Type extends MessageType {
|
|
1678
1690
|
constructor() {
|
|
1679
1691
|
super("ResumeNeighborRequest", [
|
|
@@ -1729,7 +1741,7 @@ const NodeInfoRpc = new ServiceType("NodeInfoRpc", [
|
|
|
1729
1741
|
* @generated ServiceType for protobuf service PlumtreeRpc
|
|
1730
1742
|
*/
|
|
1731
1743
|
const PlumtreeRpc = new ServiceType("PlumtreeRpc", [
|
|
1732
|
-
{ name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O:
|
|
1744
|
+
{ name: "pauseNeighbor", options: {}, I: PauseNeighborRequest, O: PauseNeighborResponse },
|
|
1733
1745
|
{ name: "resumeNeighbor", options: {}, I: ResumeNeighborRequest, O: Empty },
|
|
1734
1746
|
{ name: "sendMetadata", options: {}, I: MessageID, O: Empty }
|
|
1735
1747
|
]);
|
|
@@ -2390,12 +2402,72 @@ class NeighborUpdateManager {
|
|
|
2390
2402
|
}
|
|
2391
2403
|
}
|
|
2392
2404
|
|
|
2405
|
+
let enabled = false;
|
|
2406
|
+
function setGapDiagnosticsEnabled(val) {
|
|
2407
|
+
enabled = val;
|
|
2408
|
+
globalThis.__trackerlessGapDiagEnabled = val;
|
|
2409
|
+
}
|
|
2410
|
+
const SUMMARY_INTERVAL_MS = 2000;
|
|
2411
|
+
const accumulators = new Map();
|
|
2412
|
+
function logGapDiagnosticSampled(layer, opts = {}) {
|
|
2413
|
+
if (!enabled)
|
|
2414
|
+
return;
|
|
2415
|
+
const now = performance.now();
|
|
2416
|
+
const threshold = opts.outlierThresholdMs ?? 30;
|
|
2417
|
+
let acc = accumulators.get(layer);
|
|
2418
|
+
if (acc === undefined) {
|
|
2419
|
+
acc = { count: 0, sumDeltaMs: 0, maxDeltaMs: 0, outlierCount: 0, lastReportMs: now, lastEventMs: now };
|
|
2420
|
+
accumulators.set(layer, acc);
|
|
2421
|
+
return;
|
|
2422
|
+
}
|
|
2423
|
+
const deltaMs = now - acc.lastEventMs;
|
|
2424
|
+
acc.lastEventMs = now;
|
|
2425
|
+
acc.count++;
|
|
2426
|
+
if (deltaMs > acc.maxDeltaMs)
|
|
2427
|
+
acc.maxDeltaMs = deltaMs;
|
|
2428
|
+
acc.sumDeltaMs += deltaMs;
|
|
2429
|
+
if (deltaMs > threshold)
|
|
2430
|
+
acc.outlierCount++;
|
|
2431
|
+
if (deltaMs > threshold) {
|
|
2432
|
+
const payload = {
|
|
2433
|
+
layer,
|
|
2434
|
+
timestampMs: now,
|
|
2435
|
+
deltaMs: +deltaMs.toFixed(2),
|
|
2436
|
+
detail: opts.detail,
|
|
2437
|
+
};
|
|
2438
|
+
// eslint-disable-next-line no-console
|
|
2439
|
+
console.log('[gap-diagnostics]', JSON.stringify(payload));
|
|
2440
|
+
}
|
|
2441
|
+
if (now - acc.lastReportMs >= SUMMARY_INTERVAL_MS) {
|
|
2442
|
+
const summaryDetail = {
|
|
2443
|
+
count: acc.count,
|
|
2444
|
+
meanDeltaMs: acc.count > 0 ? +(acc.sumDeltaMs / acc.count).toFixed(2) : 0,
|
|
2445
|
+
maxDeltaMs: +acc.maxDeltaMs.toFixed(2),
|
|
2446
|
+
outlierCount: acc.outlierCount,
|
|
2447
|
+
periodMs: +(now - acc.lastReportMs).toFixed(1),
|
|
2448
|
+
};
|
|
2449
|
+
const summary = {
|
|
2450
|
+
layer: `${layer}.summary`,
|
|
2451
|
+
timestampMs: now,
|
|
2452
|
+
detail: summaryDetail,
|
|
2453
|
+
};
|
|
2454
|
+
// eslint-disable-next-line no-console
|
|
2455
|
+
console.log('[gap-diagnostics]', JSON.stringify(summary));
|
|
2456
|
+
acc.count = 0;
|
|
2457
|
+
acc.sumDeltaMs = 0;
|
|
2458
|
+
acc.maxDeltaMs = 0;
|
|
2459
|
+
acc.outlierCount = 0;
|
|
2460
|
+
acc.lastReportMs = now;
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
|
|
2393
2464
|
class ContentDeliveryRpcLocal {
|
|
2394
2465
|
options;
|
|
2395
2466
|
constructor(options) {
|
|
2396
2467
|
this.options = options;
|
|
2397
2468
|
}
|
|
2398
2469
|
async sendStreamMessage(message, context) {
|
|
2470
|
+
logGapDiagnosticSampled('trackerless.rpcLocal.sendStreamMessage');
|
|
2399
2471
|
const previousNode = context.incomingSourceDescriptor;
|
|
2400
2472
|
const previousNodeId = toNodeId(previousNode);
|
|
2401
2473
|
this.options.markForInspection(previousNodeId, message.messageId);
|
|
@@ -2756,11 +2828,16 @@ class ContentDeliveryLayerNode extends EventEmitter {
|
|
|
2756
2828
|
this.options.neighborFinder.stop();
|
|
2757
2829
|
this.options.neighborUpdateManager.stop();
|
|
2758
2830
|
this.options.inspector.stop();
|
|
2831
|
+
this.duplicateDetectors.clear();
|
|
2759
2832
|
}
|
|
2760
2833
|
broadcast(msg, previousNode) {
|
|
2761
2834
|
if (!previousNode) {
|
|
2835
|
+
logGapDiagnosticSampled('trackerless.cdNode.broadcastOut');
|
|
2762
2836
|
markAndCheckDuplicate(this.duplicateDetectors, msg.messageId, msg.previousMessageRef);
|
|
2763
2837
|
}
|
|
2838
|
+
else {
|
|
2839
|
+
logGapDiagnosticSampled('trackerless.cdNode.broadcastIn');
|
|
2840
|
+
}
|
|
2764
2841
|
this.emit('message', msg);
|
|
2765
2842
|
const skipBackPropagation = previousNode !== undefined && !this.options.temporaryConnectionRpcLocal.hasNode(previousNode);
|
|
2766
2843
|
this.options.propagation.feedUnseenMessage(msg, this.getPropagationTargets(msg), skipBackPropagation ? previousNode : null);
|
|
@@ -3058,6 +3135,7 @@ class Propagation {
|
|
|
3058
3135
|
}
|
|
3059
3136
|
sendAndAwaitThenMark({ message, source, handledNeighbors }, neighborId) {
|
|
3060
3137
|
if (!handledNeighbors.has(neighborId) && neighborId !== source) {
|
|
3138
|
+
logGapDiagnosticSampled('trackerless.propagation.sendToNeighbor');
|
|
3061
3139
|
(async () => {
|
|
3062
3140
|
try {
|
|
3063
3141
|
await this.sendToNeighbor(neighborId, message);
|
|
@@ -3331,14 +3409,17 @@ class PlumtreeRpcLocal {
|
|
|
3331
3409
|
async pauseNeighbor(request, context) {
|
|
3332
3410
|
const sender = toNodeId(context.incomingSourceDescriptor);
|
|
3333
3411
|
if (this.neighbors.has(sender)) {
|
|
3334
|
-
this.pausedNodes.add(sender, request.messageChainId);
|
|
3412
|
+
const accepted = this.pausedNodes.add(sender, request.messageChainId);
|
|
3413
|
+
return { accepted };
|
|
3335
3414
|
}
|
|
3336
|
-
return
|
|
3415
|
+
return { accepted: false };
|
|
3337
3416
|
}
|
|
3338
3417
|
async resumeNeighbor(request, context) {
|
|
3339
3418
|
const sender = context.incomingSourceDescriptor;
|
|
3340
|
-
this.
|
|
3341
|
-
|
|
3419
|
+
if (this.neighbors.has(toNodeId(sender))) {
|
|
3420
|
+
this.pausedNodes.delete(toNodeId(sender), request.messageChainId);
|
|
3421
|
+
await this.sendBuffer(request.fromTimestamp, request.messageChainId, sender);
|
|
3422
|
+
}
|
|
3342
3423
|
return Empty;
|
|
3343
3424
|
}
|
|
3344
3425
|
}
|
|
@@ -3351,10 +3432,9 @@ class PlumtreeRpcRemote extends RpcRemote {
|
|
|
3351
3432
|
await this.getClient().sendMetadata(msg, options);
|
|
3352
3433
|
}
|
|
3353
3434
|
async pauseNeighbor(messageChainId) {
|
|
3354
|
-
const options = this.formDhtRpcOptions(
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
await this.getClient().pauseNeighbor({ messageChainId }, options);
|
|
3435
|
+
const options = this.formDhtRpcOptions();
|
|
3436
|
+
const response = await this.getClient().pauseNeighbor({ messageChainId }, options);
|
|
3437
|
+
return response.accepted;
|
|
3358
3438
|
}
|
|
3359
3439
|
async resumeNeighbor(fromTimestamp, messageChainId) {
|
|
3360
3440
|
const options = this.formDhtRpcOptions({
|
|
@@ -3376,9 +3456,10 @@ class PausedNeighbors {
|
|
|
3376
3456
|
this.pausedNeighbors.set(msgChainId, new Set());
|
|
3377
3457
|
}
|
|
3378
3458
|
if (this.pausedNeighbors.get(msgChainId).size >= this.limit) {
|
|
3379
|
-
return;
|
|
3459
|
+
return false;
|
|
3380
3460
|
}
|
|
3381
3461
|
this.pausedNeighbors.get(msgChainId).add(node);
|
|
3462
|
+
return true;
|
|
3382
3463
|
}
|
|
3383
3464
|
delete(node, msgChainId) {
|
|
3384
3465
|
this.pausedNeighbors.get(msgChainId)?.delete(node);
|
|
@@ -3411,19 +3492,24 @@ class PausedNeighbors {
|
|
|
3411
3492
|
}
|
|
3412
3493
|
|
|
3413
3494
|
const MAX_PAUSED_NEIGHBORS_DEFAULT = 3;
|
|
3495
|
+
const DEFAULT_RECOVERY_TIMEOUT = 500;
|
|
3496
|
+
const DEFAULT_RECOVERY_CHECK_INTERVAL = 200;
|
|
3497
|
+
const DEFAULT_RECOVERY_COOLDOWN = 2500;
|
|
3414
3498
|
const logger$4 = new Logger('PlumtreeManager');
|
|
3415
3499
|
class PlumtreeManager extends EventEmitter {
|
|
3416
3500
|
neighbors;
|
|
3417
3501
|
localPeerDescriptor;
|
|
3418
|
-
// We have paused sending real data to these neighbrs and only send metadata
|
|
3419
3502
|
localPausedNeighbors;
|
|
3420
|
-
// We have asked these nodes to pause sending real data to us, used to limit sending of pausing and resuming requests
|
|
3421
3503
|
remotePausedNeighbors;
|
|
3422
3504
|
rpcLocal;
|
|
3423
3505
|
latestMessages = new Map();
|
|
3424
3506
|
rpcCommunicator;
|
|
3425
|
-
metadataTimestampsAheadOfRealData = new Map();
|
|
3426
3507
|
maxPausedNeighbors;
|
|
3508
|
+
recoveryState = new Map();
|
|
3509
|
+
recoveryCooldownUntil = new Map();
|
|
3510
|
+
recoveryTimeout;
|
|
3511
|
+
recoveryCooldown;
|
|
3512
|
+
abortController = new AbortController();
|
|
3427
3513
|
constructor(options) {
|
|
3428
3514
|
super();
|
|
3429
3515
|
this.neighbors = options.neighbors;
|
|
@@ -3431,12 +3517,22 @@ class PlumtreeManager extends EventEmitter {
|
|
|
3431
3517
|
this.localPeerDescriptor = options.localPeerDescriptor;
|
|
3432
3518
|
this.localPausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
|
|
3433
3519
|
this.remotePausedNeighbors = new PausedNeighbors(options.maxPausedNeighbors ?? MAX_PAUSED_NEIGHBORS_DEFAULT);
|
|
3520
|
+
this.recoveryTimeout = options.recoveryTimeout ?? DEFAULT_RECOVERY_TIMEOUT;
|
|
3521
|
+
this.recoveryCooldown = options.recoveryCooldown ?? DEFAULT_RECOVERY_COOLDOWN;
|
|
3434
3522
|
this.rpcLocal = new PlumtreeRpcLocal(this.neighbors, this.localPausedNeighbors, (metadata, previousNode) => this.onMetadata(metadata, previousNode), (fromTimestamp, msgChainId, remotePeerDescriptor) => this.sendBuffer(fromTimestamp, msgChainId, remotePeerDescriptor));
|
|
3435
|
-
this.neighbors.on('nodeRemoved',
|
|
3523
|
+
this.neighbors.on('nodeRemoved', this.onNeighborRemoved);
|
|
3436
3524
|
this.rpcCommunicator = options.rpcCommunicator;
|
|
3437
3525
|
this.rpcCommunicator.registerRpcNotification(MessageID, 'sendMetadata', (msg, context) => this.rpcLocal.sendMetadata(msg, context));
|
|
3438
|
-
this.rpcCommunicator.
|
|
3526
|
+
this.rpcCommunicator.registerRpcMethod(PauseNeighborRequest, PauseNeighborResponse, 'pauseNeighbor', (msg, context) => this.rpcLocal.pauseNeighbor(msg, context));
|
|
3439
3527
|
this.rpcCommunicator.registerRpcNotification(ResumeNeighborRequest, 'resumeNeighbor', (msg, context) => this.rpcLocal.resumeNeighbor(msg, context));
|
|
3528
|
+
setAbortableInterval(() => {
|
|
3529
|
+
const now = performance.now();
|
|
3530
|
+
for (const [chainId, state] of this.recoveryState) {
|
|
3531
|
+
if (now - state.metadataAheadSince >= this.recoveryTimeout && !state.resumeInProgress) {
|
|
3532
|
+
this.attemptRecovery(chainId, state, this.getLatestMessageTimestamp(chainId));
|
|
3533
|
+
}
|
|
3534
|
+
}
|
|
3535
|
+
}, options.recoveryCheckInterval ?? DEFAULT_RECOVERY_CHECK_INTERVAL, this.abortController.signal);
|
|
3440
3536
|
}
|
|
3441
3537
|
async pauseNeighbor(node, msgChainId) {
|
|
3442
3538
|
if (this.neighbors.has(toNodeId(node))
|
|
@@ -3444,8 +3540,16 @@ class PlumtreeManager extends EventEmitter {
|
|
|
3444
3540
|
&& this.remotePausedNeighbors.size(msgChainId) < this.maxPausedNeighbors) {
|
|
3445
3541
|
logger$4.debug(`Pausing neighbor ${toNodeId(node)}`);
|
|
3446
3542
|
this.remotePausedNeighbors.add(toNodeId(node), msgChainId);
|
|
3447
|
-
|
|
3448
|
-
|
|
3543
|
+
try {
|
|
3544
|
+
const remote = this.createRemote(node);
|
|
3545
|
+
const accepted = await remote.pauseNeighbor(msgChainId);
|
|
3546
|
+
if (!accepted) {
|
|
3547
|
+
this.remotePausedNeighbors.delete(toNodeId(node), msgChainId);
|
|
3548
|
+
}
|
|
3549
|
+
}
|
|
3550
|
+
catch (_e) {
|
|
3551
|
+
this.remotePausedNeighbors.delete(toNodeId(node), msgChainId);
|
|
3552
|
+
}
|
|
3449
3553
|
}
|
|
3450
3554
|
}
|
|
3451
3555
|
async resumeNeighbor(node, msgChainId, fromTimestamp) {
|
|
@@ -3456,9 +3560,15 @@ class PlumtreeManager extends EventEmitter {
|
|
|
3456
3560
|
await remote.resumeNeighbor(fromTimestamp, msgChainId);
|
|
3457
3561
|
}
|
|
3458
3562
|
}
|
|
3459
|
-
onNeighborRemoved(nodeId) {
|
|
3563
|
+
onNeighborRemoved = (nodeId) => {
|
|
3460
3564
|
this.localPausedNeighbors.deleteAll(nodeId);
|
|
3461
3565
|
this.remotePausedNeighbors.deleteAll(nodeId);
|
|
3566
|
+
for (const [_chainId, state] of this.recoveryState) {
|
|
3567
|
+
state.candidates = state.candidates.filter((c) => toNodeId(c) !== nodeId);
|
|
3568
|
+
if (state.lastAttemptedNode !== null && toNodeId(state.lastAttemptedNode) === nodeId) {
|
|
3569
|
+
state.lastAttemptedNode = null;
|
|
3570
|
+
}
|
|
3571
|
+
}
|
|
3462
3572
|
if (this.neighbors.size() > 0) {
|
|
3463
3573
|
this.remotePausedNeighbors.forEach((pausedNeighbors, msgChainId) => {
|
|
3464
3574
|
if (pausedNeighbors.size >= this.neighbors.size()) {
|
|
@@ -3468,7 +3578,7 @@ class PlumtreeManager extends EventEmitter {
|
|
|
3468
3578
|
}
|
|
3469
3579
|
});
|
|
3470
3580
|
}
|
|
3471
|
-
}
|
|
3581
|
+
};
|
|
3472
3582
|
getLatestMessageTimestamp(msgChainId) {
|
|
3473
3583
|
if (!this.latestMessages.has(msgChainId) || this.latestMessages.get(msgChainId).length === 0) {
|
|
3474
3584
|
return 0;
|
|
@@ -3478,22 +3588,61 @@ class PlumtreeManager extends EventEmitter {
|
|
|
3478
3588
|
async sendBuffer(fromTimestamp, msgChainId, neighbor) {
|
|
3479
3589
|
const remote = new ContentDeliveryRpcRemote(this.localPeerDescriptor, neighbor, this.rpcCommunicator, ContentDeliveryRpcClient);
|
|
3480
3590
|
const messages = this.latestMessages.get(msgChainId)?.filter((msg) => msg.messageId.timestamp > fromTimestamp) ?? [];
|
|
3481
|
-
|
|
3591
|
+
for (const msg of messages) {
|
|
3592
|
+
await remote.sendStreamMessage(msg);
|
|
3593
|
+
}
|
|
3482
3594
|
}
|
|
3483
3595
|
async onMetadata(msg, previousNode) {
|
|
3484
|
-
|
|
3485
|
-
|
|
3486
|
-
|
|
3487
|
-
|
|
3488
|
-
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
3596
|
+
const latestTs = this.getLatestMessageTimestamp(msg.messageChainId);
|
|
3597
|
+
if (latestTs >= msg.timestamp) {
|
|
3598
|
+
return;
|
|
3599
|
+
}
|
|
3600
|
+
const chainId = msg.messageChainId;
|
|
3601
|
+
const cooldownUntil = this.recoveryCooldownUntil.get(chainId);
|
|
3602
|
+
if (cooldownUntil !== undefined && performance.now() < cooldownUntil) {
|
|
3603
|
+
return;
|
|
3604
|
+
}
|
|
3605
|
+
let state = this.recoveryState.get(chainId);
|
|
3606
|
+
if (!state) {
|
|
3607
|
+
state = {
|
|
3608
|
+
timestampsAhead: new Set(),
|
|
3609
|
+
metadataAheadSince: performance.now(),
|
|
3610
|
+
candidates: [],
|
|
3611
|
+
lastAttemptedNode: null,
|
|
3612
|
+
resumeInProgress: false
|
|
3613
|
+
};
|
|
3614
|
+
this.recoveryState.set(chainId, state);
|
|
3615
|
+
}
|
|
3616
|
+
state.timestampsAhead.add(msg.timestamp);
|
|
3617
|
+
const nodeId = toNodeId(previousNode);
|
|
3618
|
+
const isLastAttempted = state.lastAttemptedNode !== null && toNodeId(state.lastAttemptedNode) === nodeId;
|
|
3619
|
+
if (!isLastAttempted && !state.candidates.some((c) => toNodeId(c) === nodeId)) {
|
|
3620
|
+
state.candidates.push(previousNode);
|
|
3621
|
+
}
|
|
3622
|
+
if (state.timestampsAhead.size > 1 && !state.resumeInProgress) {
|
|
3623
|
+
await this.attemptRecovery(chainId, state, latestTs);
|
|
3624
|
+
}
|
|
3625
|
+
}
|
|
3626
|
+
async attemptRecovery(chainId, state, latestTs) {
|
|
3627
|
+
const candidate = state.candidates.shift();
|
|
3628
|
+
if (!candidate) {
|
|
3629
|
+
state.metadataAheadSince = performance.now();
|
|
3630
|
+
return;
|
|
3631
|
+
}
|
|
3632
|
+
state.resumeInProgress = true;
|
|
3633
|
+
state.lastAttemptedNode = candidate;
|
|
3634
|
+
state.candidates = [];
|
|
3635
|
+
state.timestampsAhead.clear();
|
|
3636
|
+
state.metadataAheadSince = performance.now();
|
|
3637
|
+
try {
|
|
3638
|
+
const remote = this.createRemote(candidate);
|
|
3639
|
+
await remote.resumeNeighbor(latestTs, chainId);
|
|
3640
|
+
}
|
|
3641
|
+
catch (_e) {
|
|
3642
|
+
logger$4.debug('Recovery resume failed, will retry with next candidate');
|
|
3643
|
+
}
|
|
3644
|
+
finally {
|
|
3645
|
+
state.resumeInProgress = false;
|
|
3497
3646
|
}
|
|
3498
3647
|
}
|
|
3499
3648
|
createRemote(neighbor) {
|
|
@@ -3511,8 +3660,13 @@ class PlumtreeManager extends EventEmitter {
|
|
|
3511
3660
|
this.latestMessages.get(messageChainId).shift();
|
|
3512
3661
|
this.latestMessages.get(messageChainId).push(msg);
|
|
3513
3662
|
}
|
|
3514
|
-
|
|
3515
|
-
|
|
3663
|
+
const state = this.recoveryState.get(messageChainId);
|
|
3664
|
+
if (state) {
|
|
3665
|
+
if (state.lastAttemptedNode) {
|
|
3666
|
+
this.remotePausedNeighbors.delete(toNodeId(state.lastAttemptedNode), messageChainId);
|
|
3667
|
+
}
|
|
3668
|
+
this.recoveryState.delete(messageChainId);
|
|
3669
|
+
this.recoveryCooldownUntil.set(messageChainId, performance.now() + this.recoveryCooldown);
|
|
3516
3670
|
}
|
|
3517
3671
|
this.emit('message', msg);
|
|
3518
3672
|
const neighbors = this.neighbors.getAll().filter((neighbor) => toNodeId(neighbor.getPeerDescriptor()) !== previousNode);
|
|
@@ -3530,8 +3684,18 @@ class PlumtreeManager extends EventEmitter {
|
|
|
3530
3684
|
return this.localPausedNeighbors.isPaused(toNodeId(node), msgChainId)
|
|
3531
3685
|
|| this.remotePausedNeighbors.isPaused(toNodeId(node), msgChainId);
|
|
3532
3686
|
}
|
|
3687
|
+
getLocalPausedNeighbors() {
|
|
3688
|
+
return this.localPausedNeighbors;
|
|
3689
|
+
}
|
|
3690
|
+
getRemotePausedNeighbors() {
|
|
3691
|
+
return this.remotePausedNeighbors;
|
|
3692
|
+
}
|
|
3533
3693
|
stop() {
|
|
3694
|
+
this.abortController.abort();
|
|
3534
3695
|
this.neighbors.off('nodeRemoved', this.onNeighborRemoved);
|
|
3696
|
+
this.latestMessages.clear();
|
|
3697
|
+
this.recoveryState.clear();
|
|
3698
|
+
this.recoveryCooldownUntil.clear();
|
|
3535
3699
|
}
|
|
3536
3700
|
}
|
|
3537
3701
|
|
|
@@ -4473,5 +4637,5 @@ class NetworkNode {
|
|
|
4473
4637
|
}
|
|
4474
4638
|
}
|
|
4475
4639
|
|
|
4476
|
-
export { AsymmetricEncryptionType, ContentType, ControlLayerInfo, EncryptedGroupKey, EncryptionType, GroupKeyRequest, GroupKeyResponse, MessageID, MessageRef, NetworkNode, NetworkStack, ProxyDirection, SignatureType, StreamMessage, createNetworkNode, streamPartIdToDataKey };
|
|
4640
|
+
export { AsymmetricEncryptionType, ContentType, ControlLayerInfo, EncryptedGroupKey, EncryptionType, GroupKeyRequest, GroupKeyResponse, MessageID, MessageRef, NetworkNode, NetworkStack, ProxyDirection, SignatureType, StreamMessage, createNetworkNode, logGapDiagnosticSampled, setGapDiagnosticsEnabled as setTrackerlessGapDiagnosticsEnabled, streamPartIdToDataKey };
|
|
4477
4641
|
//# sourceMappingURL=exports.js.map
|