@peerbit/shared-log 13.2.9 → 13.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +90 -9
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1643 -203
- package/dist/src/index.js.map +1 -1
- package/package.json +13 -13
- package/src/index.ts +2316 -281
package/dist/src/index.js
CHANGED
|
@@ -33,7 +33,7 @@ var __runInitializers = (this && this.__runInitializers) || function (thisArg, i
|
|
|
33
33
|
return useValue ? value : void 0;
|
|
34
34
|
};
|
|
35
35
|
import { BorshError, deserialize, field, serialize, variant, } from "@dao-xyz/borsh";
|
|
36
|
-
import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
|
|
36
|
+
import { AnyBlockStore, RemoteBlocks, } from "@peerbit/blocks";
|
|
37
37
|
import { cidifyString } from "@peerbit/blocks-interface";
|
|
38
38
|
import { Cache } from "@peerbit/cache";
|
|
39
39
|
import { AccessError, Ed25519Keypair, Ed25519PublicKey, PublicSignKey, Secp256k1PublicKey, getPublicKeyFromPeerId, sha256Base64Sync, sha256Sync, toHexString, } from "@peerbit/crypto";
|
|
@@ -44,7 +44,7 @@ import { ClosedError, Program, TerminalOperationNotStartedError, } from "@peerbi
|
|
|
44
44
|
import { FanoutChannel, waitForSubscribers, } from "@peerbit/pubsub";
|
|
45
45
|
import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
|
|
46
46
|
import { RPC } from "@peerbit/rpc";
|
|
47
|
-
import { ACK_CONTROL_PRIORITY, AcknowledgeDelivery, AnyWhere, BACKGROUND_MESSAGE_PRIORITY, CONVERGENCE_MESSAGE_PRIORITY, DataMessage, MessageHeader, NotStartedError, SilentDelivery, createRequestTransportContext, } from "@peerbit/stream-interface";
|
|
47
|
+
import { ACK_CONTROL_PRIORITY, AcknowledgeDelivery, AnyWhere, BACKGROUND_MESSAGE_PRIORITY, CONVERGENCE_MESSAGE_PRIORITY, DataMessage, DeliveryError, MessageHeader, NotStartedError, SilentDelivery, createRequestTransportContext, } from "@peerbit/stream-interface";
|
|
48
48
|
import { AbortError, TimeoutError, debounceAccumulator, debounceFixedInterval, waitFor, } from "@peerbit/time";
|
|
49
49
|
import pDefer, {} from "p-defer";
|
|
50
50
|
import PQueue from "p-queue";
|
|
@@ -1381,6 +1381,46 @@ const isTransientReplicationAnnouncementError = (error, seen = new Set()) => {
|
|
|
1381
1381
|
: "";
|
|
1382
1382
|
return constructorName === "TimeoutError" || name === "TimeoutError";
|
|
1383
1383
|
};
|
|
1384
|
+
/**
|
|
1385
|
+
* Directed transport-delivery repair is allowed to retry explicit delivery
|
|
1386
|
+
* failures in addition to timeouts. A DirectStream ACK confirms receipt of the
|
|
1387
|
+
* signed envelope, not successful application by the receiver. Keep this
|
|
1388
|
+
* separate from the primary fanout classifier above so replicate() rejection
|
|
1389
|
+
* semantics remain unchanged for programming, serialization, and lifecycle
|
|
1390
|
+
* errors.
|
|
1391
|
+
*/
|
|
1392
|
+
const isTransientReplicationAnnouncementRepairError = (error, seen = new Set()) => {
|
|
1393
|
+
if (error != null &&
|
|
1394
|
+
(typeof error === "object" || typeof error === "function")) {
|
|
1395
|
+
if (seen.has(error)) {
|
|
1396
|
+
return false;
|
|
1397
|
+
}
|
|
1398
|
+
seen.add(error);
|
|
1399
|
+
}
|
|
1400
|
+
if (error instanceof DeliveryError || error instanceof TimeoutError) {
|
|
1401
|
+
return true;
|
|
1402
|
+
}
|
|
1403
|
+
const nested = error?.errors;
|
|
1404
|
+
if (Array.isArray(nested) && nested.length > 0) {
|
|
1405
|
+
return nested.every((item) => isTransientReplicationAnnouncementRepairError(item, new Set(seen)));
|
|
1406
|
+
}
|
|
1407
|
+
const cause = error?.cause;
|
|
1408
|
+
if (cause != null &&
|
|
1409
|
+
isTransientReplicationAnnouncementRepairError(cause, seen)) {
|
|
1410
|
+
return true;
|
|
1411
|
+
}
|
|
1412
|
+
const constructorName = typeof error?.constructor
|
|
1413
|
+
?.name === "string"
|
|
1414
|
+
? error.constructor.name
|
|
1415
|
+
: "";
|
|
1416
|
+
const name = typeof error?.name === "string"
|
|
1417
|
+
? error.name
|
|
1418
|
+
: "";
|
|
1419
|
+
return (constructorName === "DeliveryError" ||
|
|
1420
|
+
name === "DeliveryError" ||
|
|
1421
|
+
constructorName === "TimeoutError" ||
|
|
1422
|
+
name === "TimeoutError");
|
|
1423
|
+
};
|
|
1384
1424
|
const EMPTY_HASHES = [];
|
|
1385
1425
|
const normalizedHashValues = (hashes) => {
|
|
1386
1426
|
if (Array.isArray(hashes)) {
|
|
@@ -1442,6 +1482,22 @@ const createIndexableDomainFromResolution = (resolution) => {
|
|
|
1442
1482
|
}
|
|
1443
1483
|
throw new Error("Unsupported resolution");
|
|
1444
1484
|
};
|
|
1485
|
+
const mergeDefinedFanoutChannelOptions = (...sources) => {
|
|
1486
|
+
let merged;
|
|
1487
|
+
for (const source of sources) {
|
|
1488
|
+
if (!source) {
|
|
1489
|
+
continue;
|
|
1490
|
+
}
|
|
1491
|
+
for (const [key, value] of Object.entries(source)) {
|
|
1492
|
+
if (value === undefined) {
|
|
1493
|
+
continue;
|
|
1494
|
+
}
|
|
1495
|
+
merged ??= {};
|
|
1496
|
+
merged[key] = value;
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
return merged;
|
|
1500
|
+
};
|
|
1445
1501
|
const applySharedLogNativeDefaults = (options, defaults) => {
|
|
1446
1502
|
if (!defaults) {
|
|
1447
1503
|
return options;
|
|
@@ -1453,11 +1509,18 @@ const applySharedLogNativeDefaults = (options, defaults) => {
|
|
|
1453
1509
|
nativeWireSync: options?.sync?.nativeWireSync ?? defaults.sync?.nativeWireSync,
|
|
1454
1510
|
}
|
|
1455
1511
|
: undefined;
|
|
1512
|
+
const fanout = options?.fanout
|
|
1513
|
+
? {
|
|
1514
|
+
...options.fanout,
|
|
1515
|
+
channel: mergeDefinedFanoutChannelOptions(defaults.fanout?.channel, options.fanout.channel),
|
|
1516
|
+
}
|
|
1517
|
+
: undefined;
|
|
1456
1518
|
return {
|
|
1457
1519
|
...options,
|
|
1458
1520
|
nativeBackbone: options?.nativeBackbone ?? defaults.nativeBackbone,
|
|
1459
1521
|
nativeGraph: options?.nativeGraph ?? defaults.nativeGraph,
|
|
1460
1522
|
sync,
|
|
1523
|
+
fanout,
|
|
1461
1524
|
};
|
|
1462
1525
|
};
|
|
1463
1526
|
export const DEFAULT_MIN_REPLICAS = 2;
|
|
@@ -1478,6 +1541,14 @@ const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
|
|
|
1478
1541
|
// DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
|
|
1479
1542
|
const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
|
|
1480
1543
|
const REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL = 1000;
|
|
1544
|
+
const REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL = 1000;
|
|
1545
|
+
const REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS = 3;
|
|
1546
|
+
// Repair one bounded cohort per mutation generation. The subscriber snapshot
|
|
1547
|
+
// is a best-effort cache and can contain thousands of entries, so attempting
|
|
1548
|
+
// the whole cache after every role mutation would turn convergence repair into
|
|
1549
|
+
// an unbounded burst of separately signed, acknowledged messages. A cursor
|
|
1550
|
+
// retained across generations rotates best-effort coverage over later changes.
|
|
1551
|
+
const REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION = 8;
|
|
1481
1552
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
|
|
1482
1553
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
|
|
1483
1554
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
|
|
@@ -1517,9 +1588,13 @@ const REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT = 2;
|
|
|
1517
1588
|
const CHURN_REPAIR_RETRY_SCHEDULE_MS = [
|
|
1518
1589
|
0, 1_000, 3_000, 7_000, 15_000, 30_000, 45_000,
|
|
1519
1590
|
];
|
|
1591
|
+
// Preserve the bounded retry window for transient local misses, but serialize
|
|
1592
|
+
// delayed warmup sends per target so fixed snapshots cannot overlap and amplify
|
|
1593
|
+
// large transfers. Every queued pass re-checks current peer knowledge on entry.
|
|
1520
1594
|
const JOIN_WARMUP_RETRY_SCHEDULE_MS = [
|
|
1521
1595
|
0, 1_000, 3_000, 7_000, 15_000, 30_000, 60_000,
|
|
1522
1596
|
];
|
|
1597
|
+
const JOIN_WARMUP_SEND_SPACING_MS = 250;
|
|
1523
1598
|
const JOIN_AUTHORITATIVE_RETRY_SCHEDULE_MS = [
|
|
1524
1599
|
0, 1_000, 3_000, 7_000, 15_000, 30_000, 60_000,
|
|
1525
1600
|
];
|
|
@@ -1686,6 +1761,11 @@ let SharedLog = (() => {
|
|
|
1686
1761
|
_subscriptionChangeCallbacks;
|
|
1687
1762
|
_acceptSubscriptionChangeCallbacks = false;
|
|
1688
1763
|
_replicationLifecycleController;
|
|
1764
|
+
_activeReceiveHandlersByPeer;
|
|
1765
|
+
_receiveHandlerDrainByPeer;
|
|
1766
|
+
_receiveCleanupGateByPeer;
|
|
1767
|
+
_subscriptionOpeningEpochByPeer;
|
|
1768
|
+
_openingSyncCapabilitiesByPeer;
|
|
1689
1769
|
_onFanoutDataFn;
|
|
1690
1770
|
_onFanoutUnicastFn;
|
|
1691
1771
|
_fanoutChannel;
|
|
@@ -1695,6 +1775,7 @@ let SharedLog = (() => {
|
|
|
1695
1775
|
_closeController;
|
|
1696
1776
|
_respondToIHaveTimeout;
|
|
1697
1777
|
_checkedPrune;
|
|
1778
|
+
_pendingIHaveCallbacks;
|
|
1698
1779
|
_pendingIHaveExpiryTimer;
|
|
1699
1780
|
_pendingIHaveExpiryDeadline = Number.POSITIVE_INFINITY;
|
|
1700
1781
|
get _pendingDeletes() {
|
|
@@ -1710,6 +1791,18 @@ let SharedLog = (() => {
|
|
|
1710
1791
|
_replicationInfoBlockedPeers;
|
|
1711
1792
|
_replicationInfoRequestByPeer;
|
|
1712
1793
|
_replicationInfoApplyQueueByPeer;
|
|
1794
|
+
// Local receive generations fence replication-info handlers that were admitted
|
|
1795
|
+
// before a liveness eviction but reach the per-peer apply lane after it. Unlike
|
|
1796
|
+
// message timestamps, these tokens never compare clocks from different peers.
|
|
1797
|
+
_replicationInfoReceiveEpochByPeer;
|
|
1798
|
+
// Subscription callbacks can overlap because removing a replicator mutates the
|
|
1799
|
+
// replication index asynchronously. Keep that lifecycle separate from message
|
|
1800
|
+
// timestamps so a reconnect can synchronously revoke an older unsubscribe.
|
|
1801
|
+
_subscriptionEpochByPeer;
|
|
1802
|
+
// A superseded removal may be the queue item that actually observed an active
|
|
1803
|
+
// replicator. Carry that leave obligation to the transition that ultimately
|
|
1804
|
+
// wins, while a winning reconnect clears it without emitting a stale leave.
|
|
1805
|
+
_pendingReplicatorLeaveByPeer;
|
|
1713
1806
|
_replicatorLivenessSweepRunning;
|
|
1714
1807
|
_replicatorLivenessTimer;
|
|
1715
1808
|
_replicatorLivenessTargets;
|
|
@@ -2461,6 +2554,15 @@ let SharedLog = (() => {
|
|
|
2461
2554
|
_replicationAnnouncementRetryPending;
|
|
2462
2555
|
_replicationAnnouncementRetryGeneration;
|
|
2463
2556
|
_replicationAnnouncementRetryController;
|
|
2557
|
+
replicationAnnouncementRepairDebounced;
|
|
2558
|
+
_replicationAnnouncementRepairPending;
|
|
2559
|
+
_replicationAnnouncementRepairGeneration;
|
|
2560
|
+
_replicationAnnouncementRepairGenerationController;
|
|
2561
|
+
_replicationAnnouncementRepairTargets;
|
|
2562
|
+
_replicationAnnouncementRepairCohortSelected;
|
|
2563
|
+
_replicationAnnouncementRepairFairCursorHash;
|
|
2564
|
+
_replicationAnnouncementRepairMaxAttempts;
|
|
2565
|
+
_replicationAnnouncementRepairController;
|
|
2464
2566
|
// A fn for debouncing the calls for pruning
|
|
2465
2567
|
pruneDebouncedFn;
|
|
2466
2568
|
responseToPruneDebouncedFn;
|
|
@@ -2479,10 +2581,16 @@ let SharedLog = (() => {
|
|
|
2479
2581
|
_repairSweepRunning;
|
|
2480
2582
|
_repairSweepPendingModes;
|
|
2481
2583
|
_repairSweepPendingPeersByMode;
|
|
2584
|
+
_repairSweepJoinWarmupGenerationByTarget;
|
|
2482
2585
|
_repairFrontierByMode;
|
|
2483
2586
|
_repairFrontierActiveTargetsByMode;
|
|
2484
2587
|
_repairFrontierBypassKnownPeersByMode;
|
|
2588
|
+
_joinWarmupGenerationByTarget;
|
|
2589
|
+
_joinWarmupSendStateByTarget;
|
|
2590
|
+
_joinWarmupRetryTimersByTarget;
|
|
2591
|
+
_joinWarmupScheduledRetriesByTarget;
|
|
2485
2592
|
_repairSweepOptimisticGidPeersPending;
|
|
2593
|
+
_repairSweepOptimisticGidsByPeer;
|
|
2486
2594
|
_entryKnownPeers;
|
|
2487
2595
|
_entryKnownPeerObservedAt;
|
|
2488
2596
|
_joinAuthoritativeRepairTimersByDelay;
|
|
@@ -2527,19 +2635,36 @@ let SharedLog = (() => {
|
|
|
2527
2635
|
this.rpc = new RPC();
|
|
2528
2636
|
this._checkedPrune = new CheckedPruneCoordinator();
|
|
2529
2637
|
this._pendingIHave = new Map();
|
|
2638
|
+
this._pendingIHaveCallbacks = new Set();
|
|
2530
2639
|
this.latestReplicationInfoMessage = new Map();
|
|
2531
2640
|
this._replicationInfoBlockedPeers = new Set();
|
|
2532
2641
|
this._replicationInfoRequestByPeer = new Map();
|
|
2533
2642
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
2643
|
+
this._replicationInfoReceiveEpochByPeer = new Map();
|
|
2644
|
+
this._subscriptionEpochByPeer = new Map();
|
|
2645
|
+
this._pendingReplicatorLeaveByPeer = new Set();
|
|
2646
|
+
this._activeReceiveHandlersByPeer = new Map();
|
|
2647
|
+
this._receiveHandlerDrainByPeer = new Map();
|
|
2648
|
+
this._receiveCleanupGateByPeer = new Map();
|
|
2649
|
+
this._subscriptionOpeningEpochByPeer = new Map();
|
|
2650
|
+
this._openingSyncCapabilitiesByPeer = new Map();
|
|
2534
2651
|
this._gidPeersHistory = new Map();
|
|
2535
2652
|
this._repairRetryTimers = new Set();
|
|
2536
2653
|
this._recentRepairDispatch = new Map();
|
|
2537
2654
|
this._repairSweepRunning = false;
|
|
2538
2655
|
this._repairSweepPendingModes = new Set();
|
|
2539
2656
|
this._repairSweepPendingPeersByMode = createRepairPendingPeersByMode();
|
|
2657
|
+
this._repairSweepJoinWarmupGenerationByTarget = new Map();
|
|
2540
2658
|
this._repairFrontierByMode = createRepairFrontierByMode();
|
|
2541
2659
|
this._repairFrontierActiveTargetsByMode = createRepairActiveTargetsByMode();
|
|
2660
|
+
this._repairFrontierBypassKnownPeersByMode =
|
|
2661
|
+
createRepairFrontierBypassKnownPeersByMode();
|
|
2662
|
+
this._joinWarmupGenerationByTarget = new Map();
|
|
2663
|
+
this._joinWarmupSendStateByTarget = new Map();
|
|
2664
|
+
this._joinWarmupRetryTimersByTarget = new Map();
|
|
2665
|
+
this._joinWarmupScheduledRetriesByTarget = new Map();
|
|
2542
2666
|
this._repairSweepOptimisticGidPeersPending = new Map();
|
|
2667
|
+
this._repairSweepOptimisticGidsByPeer = new Map();
|
|
2543
2668
|
this._entryKnownPeers = new Map();
|
|
2544
2669
|
this._joinAuthoritativeRepairTimersByDelay = new Map();
|
|
2545
2670
|
this._joinAuthoritativeRepairPeersByDelay = new Map();
|
|
@@ -2562,6 +2687,16 @@ let SharedLog = (() => {
|
|
|
2562
2687
|
this._replicationAnnouncementRetryPending = false;
|
|
2563
2688
|
this._replicationAnnouncementRetryGeneration = 0;
|
|
2564
2689
|
this._replicationAnnouncementRetryController = new AbortController();
|
|
2690
|
+
this._replicationAnnouncementRepairPending = false;
|
|
2691
|
+
this._replicationAnnouncementRepairGeneration = 0;
|
|
2692
|
+
this._replicationAnnouncementRepairGenerationController =
|
|
2693
|
+
new AbortController();
|
|
2694
|
+
this._replicationAnnouncementRepairTargets = new Map();
|
|
2695
|
+
this._replicationAnnouncementRepairCohortSelected = false;
|
|
2696
|
+
this._replicationAnnouncementRepairFairCursorHash = undefined;
|
|
2697
|
+
this._replicationAnnouncementRepairMaxAttempts =
|
|
2698
|
+
REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS;
|
|
2699
|
+
this._replicationAnnouncementRepairController = new AbortController();
|
|
2565
2700
|
this.pendingMaturity = new Map();
|
|
2566
2701
|
this._closeController = new AbortController();
|
|
2567
2702
|
}
|
|
@@ -2848,7 +2983,7 @@ let SharedLog = (() => {
|
|
|
2848
2983
|
});
|
|
2849
2984
|
}
|
|
2850
2985
|
async _sendAckWithUnifiedHints(properties) {
|
|
2851
|
-
const { peer, message, payload, fanoutUnicastOptions } = properties;
|
|
2986
|
+
const { peer, message, payload, priority, fanoutUnicastOptions } = properties;
|
|
2852
2987
|
const hints = await this._getSortedRouteHints(peer);
|
|
2853
2988
|
const hasDirectHint = hints.some((hint) => hint.kind === "directstream-ack");
|
|
2854
2989
|
const fanoutHint = hints.find((hint) => hint.kind === "fanout-token");
|
|
@@ -2859,6 +2994,7 @@ let SharedLog = (() => {
|
|
|
2859
2994
|
redundancy: 1,
|
|
2860
2995
|
to: [peer],
|
|
2861
2996
|
}),
|
|
2997
|
+
priority,
|
|
2862
2998
|
});
|
|
2863
2999
|
return;
|
|
2864
3000
|
}
|
|
@@ -2889,6 +3025,7 @@ let SharedLog = (() => {
|
|
|
2889
3025
|
redundancy: 1,
|
|
2890
3026
|
to: [peer],
|
|
2891
3027
|
}),
|
|
3028
|
+
priority,
|
|
2892
3029
|
});
|
|
2893
3030
|
}
|
|
2894
3031
|
/** Live append gossip that stayed on the plain TS path (countable in tests). */
|
|
@@ -3202,6 +3339,7 @@ let SharedLog = (() => {
|
|
|
3202
3339
|
peer,
|
|
3203
3340
|
message,
|
|
3204
3341
|
payload,
|
|
3342
|
+
priority: delivery.priority,
|
|
3205
3343
|
fanoutUnicastOptions,
|
|
3206
3344
|
});
|
|
3207
3345
|
})());
|
|
@@ -3214,6 +3352,7 @@ let SharedLog = (() => {
|
|
|
3214
3352
|
redundancy: 1,
|
|
3215
3353
|
to: nativeDeliveryPlan.silentTo,
|
|
3216
3354
|
}),
|
|
3355
|
+
priority: delivery.priority,
|
|
3217
3356
|
})
|
|
3218
3357
|
.catch((error) => logger.error(error));
|
|
3219
3358
|
}
|
|
@@ -3337,6 +3476,7 @@ let SharedLog = (() => {
|
|
|
3337
3476
|
peer,
|
|
3338
3477
|
message,
|
|
3339
3478
|
payload,
|
|
3479
|
+
priority: delivery.priority,
|
|
3340
3480
|
fanoutUnicastOptions,
|
|
3341
3481
|
});
|
|
3342
3482
|
})());
|
|
@@ -3346,6 +3486,7 @@ let SharedLog = (() => {
|
|
|
3346
3486
|
this.rpc
|
|
3347
3487
|
.send(message, {
|
|
3348
3488
|
mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
|
|
3489
|
+
priority: delivery.priority,
|
|
3349
3490
|
})
|
|
3350
3491
|
.catch((error) => logger.error(error));
|
|
3351
3492
|
}
|
|
@@ -3611,6 +3752,134 @@ let SharedLog = (() => {
|
|
|
3611
3752
|
await Promise.all([...callbacks]);
|
|
3612
3753
|
}
|
|
3613
3754
|
}
|
|
3755
|
+
isPeerReceiveAdmissionOpen(peerHash, replicationLifecycleController, subscriptionEpoch, options) {
|
|
3756
|
+
return (this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
3757
|
+
this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch) &&
|
|
3758
|
+
(options?.allowReplicationInfoBlocked === true ||
|
|
3759
|
+
!this._replicationInfoBlockedPeers.has(peerHash)) &&
|
|
3760
|
+
(options?.allowCleanupGate === true ||
|
|
3761
|
+
(this._receiveCleanupGateByPeer.get(peerHash) ?? 0) === 0));
|
|
3762
|
+
}
|
|
3763
|
+
acquirePeerReceiveLease(peerHash, replicationLifecycleController, subscriptionEpoch, options) {
|
|
3764
|
+
if (!this.isPeerReceiveAdmissionOpen(peerHash, replicationLifecycleController, subscriptionEpoch, options)) {
|
|
3765
|
+
return;
|
|
3766
|
+
}
|
|
3767
|
+
let state = this._activeReceiveHandlersByPeer.get(peerHash);
|
|
3768
|
+
if (!state) {
|
|
3769
|
+
const current = { active: 0 };
|
|
3770
|
+
state = { current, activeBuckets: new Set() };
|
|
3771
|
+
this._activeReceiveHandlersByPeer.set(peerHash, state);
|
|
3772
|
+
}
|
|
3773
|
+
const bucket = state.current;
|
|
3774
|
+
bucket.active += 1;
|
|
3775
|
+
state.activeBuckets.add(bucket);
|
|
3776
|
+
let released = false;
|
|
3777
|
+
return () => {
|
|
3778
|
+
if (released) {
|
|
3779
|
+
return;
|
|
3780
|
+
}
|
|
3781
|
+
released = true;
|
|
3782
|
+
bucket.active -= 1;
|
|
3783
|
+
if (bucket.active > 0) {
|
|
3784
|
+
return;
|
|
3785
|
+
}
|
|
3786
|
+
state.activeBuckets.delete(bucket);
|
|
3787
|
+
bucket.drain?.resolve();
|
|
3788
|
+
if (state.activeBuckets.size === 0 &&
|
|
3789
|
+
state.current.active === 0 &&
|
|
3790
|
+
this._activeReceiveHandlersByPeer.get(peerHash) === state) {
|
|
3791
|
+
this._activeReceiveHandlersByPeer.delete(peerHash);
|
|
3792
|
+
}
|
|
3793
|
+
};
|
|
3794
|
+
}
|
|
3795
|
+
async drainPeerReceiveHandlers(peerHash) {
|
|
3796
|
+
const state = this._activeReceiveHandlersByPeer.get(peerHash);
|
|
3797
|
+
if (!state || state.activeBuckets.size === 0) {
|
|
3798
|
+
return;
|
|
3799
|
+
}
|
|
3800
|
+
// Rotate before awaiting so a reconnect/opening generation can keep receiving
|
|
3801
|
+
// sync traffic without joining the drain for the previous subscription. Cleanup
|
|
3802
|
+
// callers gate admission first; terminal callers also repeat until empty.
|
|
3803
|
+
const buckets = [...state.activeBuckets];
|
|
3804
|
+
state.current = { active: 0 };
|
|
3805
|
+
const drain = Promise.all(buckets.map((bucket) => {
|
|
3806
|
+
bucket.drain ??= pDefer();
|
|
3807
|
+
return bucket.drain.promise;
|
|
3808
|
+
})).then(() => undefined);
|
|
3809
|
+
let drains = this._receiveHandlerDrainByPeer.get(peerHash);
|
|
3810
|
+
if (!drains) {
|
|
3811
|
+
drains = new Set();
|
|
3812
|
+
this._receiveHandlerDrainByPeer.set(peerHash, drains);
|
|
3813
|
+
}
|
|
3814
|
+
drains.add(drain);
|
|
3815
|
+
try {
|
|
3816
|
+
await drain;
|
|
3817
|
+
}
|
|
3818
|
+
finally {
|
|
3819
|
+
drains.delete(drain);
|
|
3820
|
+
if (drains.size === 0) {
|
|
3821
|
+
this._receiveHandlerDrainByPeer.delete(peerHash);
|
|
3822
|
+
}
|
|
3823
|
+
}
|
|
3824
|
+
}
|
|
3825
|
+
async drainReceiveHandlers() {
|
|
3826
|
+
for (;;) {
|
|
3827
|
+
const peers = [...this._activeReceiveHandlersByPeer.keys()];
|
|
3828
|
+
if (peers.length === 0) {
|
|
3829
|
+
return;
|
|
3830
|
+
}
|
|
3831
|
+
await Promise.all(peers.map((peerHash) => this.drainPeerReceiveHandlers(peerHash)));
|
|
3832
|
+
}
|
|
3833
|
+
}
|
|
3834
|
+
runPendingIHaveCallback(pending, entry) {
|
|
3835
|
+
const replicationLifecycleController = this._replicationLifecycleController;
|
|
3836
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
3837
|
+
if (this._pendingIHave.get(entry.hash) === pending) {
|
|
3838
|
+
pending.clear();
|
|
3839
|
+
this._pendingIHave.delete(entry.hash);
|
|
3840
|
+
}
|
|
3841
|
+
return;
|
|
3842
|
+
}
|
|
3843
|
+
// Register before invoking the callback so a synchronous terminal reentry
|
|
3844
|
+
// cannot make close/drop miss work that has already been admitted.
|
|
3845
|
+
const completion = pDefer();
|
|
3846
|
+
const observed = completion.promise.catch((error) => {
|
|
3847
|
+
if (!(this.isTerminating() && isNotStartedError(error))) {
|
|
3848
|
+
logger.error(error?.toString?.() ?? String(error));
|
|
3849
|
+
}
|
|
3850
|
+
});
|
|
3851
|
+
this._pendingIHaveCallbacks.add(observed);
|
|
3852
|
+
void observed.finally(() => {
|
|
3853
|
+
this._pendingIHaveCallbacks.delete(observed);
|
|
3854
|
+
if (this._pendingIHave.get(entry.hash) === pending) {
|
|
3855
|
+
pending.clear();
|
|
3856
|
+
this._pendingIHave.delete(entry.hash);
|
|
3857
|
+
}
|
|
3858
|
+
});
|
|
3859
|
+
try {
|
|
3860
|
+
Promise.resolve(pending.callback(entry)).then(() => completion.resolve(), (error) => completion.reject(error));
|
|
3861
|
+
}
|
|
3862
|
+
catch (error) {
|
|
3863
|
+
completion.reject(error);
|
|
3864
|
+
}
|
|
3865
|
+
}
|
|
3866
|
+
async drainPendingIHaveCallbacks() {
|
|
3867
|
+
while (this._pendingIHaveCallbacks.size > 0) {
|
|
3868
|
+
await Promise.all([...this._pendingIHaveCallbacks]);
|
|
3869
|
+
}
|
|
3870
|
+
}
|
|
3871
|
+
blockPeerReceiveAdmission(peerHash) {
|
|
3872
|
+
this._receiveCleanupGateByPeer.set(peerHash, (this._receiveCleanupGateByPeer.get(peerHash) ?? 0) + 1);
|
|
3873
|
+
}
|
|
3874
|
+
unblockPeerReceiveAdmission(peerHash) {
|
|
3875
|
+
const remaining = (this._receiveCleanupGateByPeer.get(peerHash) ?? 1) - 1;
|
|
3876
|
+
if (remaining > 0) {
|
|
3877
|
+
this._receiveCleanupGateByPeer.set(peerHash, remaining);
|
|
3878
|
+
}
|
|
3879
|
+
else {
|
|
3880
|
+
this._receiveCleanupGateByPeer.delete(peerHash);
|
|
3881
|
+
}
|
|
3882
|
+
}
|
|
3614
3883
|
handleReplicationLifecycleSendError(error, controller = this._replicationLifecycleController) {
|
|
3615
3884
|
if ((controller?.signal.aborted ||
|
|
3616
3885
|
!this.isReplicationLifecycleActive(controller)) &&
|
|
@@ -3689,11 +3958,232 @@ let SharedLog = (() => {
|
|
|
3689
3958
|
},
|
|
3690
3959
|
});
|
|
3691
3960
|
}
|
|
3961
|
+
setupReplicationAnnouncementRepairFunction(interval = REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL, maxAttempts = REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS) {
|
|
3962
|
+
if (!Number.isSafeInteger(maxAttempts) || maxAttempts <= 0) {
|
|
3963
|
+
throw new RangeError("Replication announcement repair attempts must be positive");
|
|
3964
|
+
}
|
|
3965
|
+
this.replicationAnnouncementRepairDebounced?.close();
|
|
3966
|
+
this._replicationAnnouncementRepairController?.abort();
|
|
3967
|
+
this._replicationAnnouncementRepairGenerationController?.abort();
|
|
3968
|
+
this._replicationAnnouncementRepairController = new AbortController();
|
|
3969
|
+
this._replicationAnnouncementRepairGenerationController =
|
|
3970
|
+
new AbortController();
|
|
3971
|
+
this._replicationAnnouncementRepairPending = false;
|
|
3972
|
+
this._replicationAnnouncementRepairGeneration =
|
|
3973
|
+
this._replicationAnnouncementRetryGeneration;
|
|
3974
|
+
this._replicationAnnouncementRepairTargets = new Map();
|
|
3975
|
+
this._replicationAnnouncementRepairCohortSelected = false;
|
|
3976
|
+
this._replicationAnnouncementRepairFairCursorHash = undefined;
|
|
3977
|
+
this._replicationAnnouncementRepairMaxAttempts = maxAttempts;
|
|
3978
|
+
this.replicationAnnouncementRepairDebounced = debounceFixedInterval(() => this.runCurrentReplicationStateAnnouncementRepair(), interval, {
|
|
3979
|
+
leading: false,
|
|
3980
|
+
// The wrapper catches worker failures while it still owns the generation
|
|
3981
|
+
// context. Keep this boundary visibility-only: it must never mutate a
|
|
3982
|
+
// possibly newer generation's pending state.
|
|
3983
|
+
onError: (error) => logger.error(error),
|
|
3984
|
+
});
|
|
3985
|
+
}
|
|
3986
|
+
cancelCurrentReplicationStateAnnouncementRepair() {
|
|
3987
|
+
this._replicationAnnouncementRepairPending = false;
|
|
3988
|
+
this._replicationAnnouncementRepairController?.abort();
|
|
3989
|
+
this._replicationAnnouncementRepairGenerationController?.abort();
|
|
3990
|
+
this.replicationAnnouncementRepairDebounced?.close();
|
|
3991
|
+
this._replicationAnnouncementRepairTargets?.clear();
|
|
3992
|
+
}
|
|
3993
|
+
advanceCurrentReplicationStateAnnouncementRepairGeneration() {
|
|
3994
|
+
const generation = this._replicationAnnouncementRetryGeneration;
|
|
3995
|
+
if (generation === this._replicationAnnouncementRepairGeneration) {
|
|
3996
|
+
return;
|
|
3997
|
+
}
|
|
3998
|
+
// Abort acknowledged sends carrying the old full-state snapshot before the
|
|
3999
|
+
// primary announcement for the new mutation waits on transport. Otherwise a
|
|
4000
|
+
// stale batch can hold the current state behind DirectStream's seek timeout.
|
|
4001
|
+
this._replicationAnnouncementRepairGenerationController?.abort();
|
|
4002
|
+
this._replicationAnnouncementRepairGenerationController =
|
|
4003
|
+
new AbortController();
|
|
4004
|
+
this._replicationAnnouncementRepairGeneration = generation;
|
|
4005
|
+
this._replicationAnnouncementRepairPending = false;
|
|
4006
|
+
this._replicationAnnouncementRepairTargets.clear();
|
|
4007
|
+
this._replicationAnnouncementRepairCohortSelected = false;
|
|
4008
|
+
}
|
|
4009
|
+
queueCurrentReplicationStateAnnouncementRepair() {
|
|
4010
|
+
if (this.closed ||
|
|
4011
|
+
this._closeController.signal.aborted ||
|
|
4012
|
+
this._replicationAnnouncementRepairController.signal.aborted ||
|
|
4013
|
+
!this.replicationAnnouncementRepairDebounced) {
|
|
4014
|
+
return;
|
|
4015
|
+
}
|
|
4016
|
+
this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
|
|
4017
|
+
this._replicationAnnouncementRepairPending = true;
|
|
4018
|
+
void this.replicationAnnouncementRepairDebounced.call();
|
|
4019
|
+
}
|
|
4020
|
+
async runCurrentReplicationStateAnnouncementRepair() {
|
|
4021
|
+
const generation = this._replicationAnnouncementRetryGeneration;
|
|
4022
|
+
const lifecycleController = this._replicationAnnouncementRepairController;
|
|
4023
|
+
const generationController = this._replicationAnnouncementRepairGenerationController;
|
|
4024
|
+
try {
|
|
4025
|
+
await this.repairCurrentReplicationStateAnnouncement({
|
|
4026
|
+
generation,
|
|
4027
|
+
lifecycleController,
|
|
4028
|
+
generationController,
|
|
4029
|
+
});
|
|
4030
|
+
}
|
|
4031
|
+
catch (error) {
|
|
4032
|
+
if (this.closed ||
|
|
4033
|
+
this._closeController.signal.aborted ||
|
|
4034
|
+
lifecycleController.signal.aborted ||
|
|
4035
|
+
generationController.signal.aborted ||
|
|
4036
|
+
generation !== this._replicationAnnouncementRetryGeneration ||
|
|
4037
|
+
generationController !==
|
|
4038
|
+
this._replicationAnnouncementRepairGenerationController) {
|
|
4039
|
+
return;
|
|
4040
|
+
}
|
|
4041
|
+
if (isNotStartedError(error)) {
|
|
4042
|
+
return;
|
|
4043
|
+
}
|
|
4044
|
+
// Only the worker that still owns the current generation may conclude
|
|
4045
|
+
// that its repair failed. A stale worker must not clear a newer call's
|
|
4046
|
+
// pending flag or attribute its error to the new generation.
|
|
4047
|
+
this._replicationAnnouncementRepairPending = false;
|
|
4048
|
+
logger.error(error);
|
|
4049
|
+
}
|
|
4050
|
+
}
|
|
4051
|
+
async repairCurrentReplicationStateAnnouncement(context) {
|
|
4052
|
+
if (!this._replicationAnnouncementRepairPending) {
|
|
4053
|
+
return;
|
|
4054
|
+
}
|
|
4055
|
+
const generation = context?.generation ?? this._replicationAnnouncementRetryGeneration;
|
|
4056
|
+
const lifecycleController = context?.lifecycleController ??
|
|
4057
|
+
this._replicationAnnouncementRepairController;
|
|
4058
|
+
const generationController = context?.generationController ??
|
|
4059
|
+
this._replicationAnnouncementRepairGenerationController;
|
|
4060
|
+
const segments = (await this.getMyReplicationSegments()).map((range) => range.toReplicationRange());
|
|
4061
|
+
if (this.closed ||
|
|
4062
|
+
this._closeController.signal.aborted ||
|
|
4063
|
+
lifecycleController.signal.aborted ||
|
|
4064
|
+
generationController.signal.aborted) {
|
|
4065
|
+
return;
|
|
4066
|
+
}
|
|
4067
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
4068
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
4069
|
+
return;
|
|
4070
|
+
}
|
|
4071
|
+
const subscribers = (await this.node.services.pubsub.getSubscribers(this.topic)) ?? [];
|
|
4072
|
+
if (this.closed ||
|
|
4073
|
+
this._closeController.signal.aborted ||
|
|
4074
|
+
lifecycleController.signal.aborted ||
|
|
4075
|
+
generationController.signal.aborted) {
|
|
4076
|
+
return;
|
|
4077
|
+
}
|
|
4078
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
4079
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
4080
|
+
return;
|
|
4081
|
+
}
|
|
4082
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
4083
|
+
const currentTargets = new Map();
|
|
4084
|
+
for (const key of subscribers) {
|
|
4085
|
+
const hash = key.hashcode();
|
|
4086
|
+
if (hash !== selfHash &&
|
|
4087
|
+
!this._replicationInfoBlockedPeers.has(hash) &&
|
|
4088
|
+
!currentTargets.has(hash)) {
|
|
4089
|
+
currentTargets.set(hash, key);
|
|
4090
|
+
}
|
|
4091
|
+
}
|
|
4092
|
+
for (const [hash, target] of this._replicationAnnouncementRepairTargets) {
|
|
4093
|
+
if (target.generation !== generation || !currentTargets.has(hash)) {
|
|
4094
|
+
this._replicationAnnouncementRepairTargets.delete(hash);
|
|
4095
|
+
}
|
|
4096
|
+
else {
|
|
4097
|
+
target.key = currentTargets.get(hash);
|
|
4098
|
+
}
|
|
4099
|
+
}
|
|
4100
|
+
if (!this._replicationAnnouncementRepairCohortSelected) {
|
|
4101
|
+
const candidates = [...currentTargets.entries()].sort(([left], [right]) => left.localeCompare(right));
|
|
4102
|
+
const cursorIndex = this._replicationAnnouncementRepairFairCursorHash
|
|
4103
|
+
? candidates.findIndex(([hash]) => hash.localeCompare(this._replicationAnnouncementRepairFairCursorHash) > 0)
|
|
4104
|
+
: 0;
|
|
4105
|
+
const fairStart = cursorIndex < 0 ? 0 : cursorIndex;
|
|
4106
|
+
const fairOrder = [
|
|
4107
|
+
...candidates.slice(fairStart),
|
|
4108
|
+
...candidates.slice(0, fairStart),
|
|
4109
|
+
];
|
|
4110
|
+
const cohort = fairOrder.slice(0, REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION);
|
|
4111
|
+
for (const [hash, key] of cohort) {
|
|
4112
|
+
this._replicationAnnouncementRepairTargets.set(hash, {
|
|
4113
|
+
key,
|
|
4114
|
+
generation,
|
|
4115
|
+
attempts: 0,
|
|
4116
|
+
done: false,
|
|
4117
|
+
});
|
|
4118
|
+
}
|
|
4119
|
+
if (cohort.length > 0) {
|
|
4120
|
+
this._replicationAnnouncementRepairFairCursorHash =
|
|
4121
|
+
cohort[cohort.length - 1][0];
|
|
4122
|
+
}
|
|
4123
|
+
this._replicationAnnouncementRepairCohortSelected = true;
|
|
4124
|
+
}
|
|
4125
|
+
const batch = [
|
|
4126
|
+
...this._replicationAnnouncementRepairTargets.entries(),
|
|
4127
|
+
].filter(([, target]) => !target.done);
|
|
4128
|
+
const snapshot = new AllReplicatingSegmentsMessage({ segments });
|
|
4129
|
+
const results = await Promise.allSettled(batch.map(([, target]) => this.rpc.send(snapshot, {
|
|
4130
|
+
mode: new AcknowledgeDelivery({
|
|
4131
|
+
to: [target.key],
|
|
4132
|
+
redundancy: 1,
|
|
4133
|
+
}),
|
|
4134
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
4135
|
+
signal: generationController.signal,
|
|
4136
|
+
})));
|
|
4137
|
+
if (this.closed ||
|
|
4138
|
+
this._closeController.signal.aborted ||
|
|
4139
|
+
lifecycleController.signal.aborted ||
|
|
4140
|
+
generationController.signal.aborted) {
|
|
4141
|
+
return;
|
|
4142
|
+
}
|
|
4143
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
4144
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
4145
|
+
return;
|
|
4146
|
+
}
|
|
4147
|
+
for (const [index, result] of results.entries()) {
|
|
4148
|
+
const [hash, attemptedTarget] = batch[index];
|
|
4149
|
+
const target = this._replicationAnnouncementRepairTargets.get(hash);
|
|
4150
|
+
if (target !== attemptedTarget || target.generation !== generation) {
|
|
4151
|
+
continue;
|
|
4152
|
+
}
|
|
4153
|
+
if (result.status === "fulfilled") {
|
|
4154
|
+
// DirectStream ACKs confirm that the signed transport envelope reached
|
|
4155
|
+
// the target. Applying the contained replication state remains a
|
|
4156
|
+
// receiver-local, best-effort operation.
|
|
4157
|
+
target.done = true;
|
|
4158
|
+
continue;
|
|
4159
|
+
}
|
|
4160
|
+
target.attempts += 1;
|
|
4161
|
+
if (!isTransientReplicationAnnouncementRepairError(result.reason)) {
|
|
4162
|
+
target.done = true;
|
|
4163
|
+
logger.error(result.reason);
|
|
4164
|
+
}
|
|
4165
|
+
else if (target.attempts >= this._replicationAnnouncementRepairMaxAttempts) {
|
|
4166
|
+
target.done = true;
|
|
4167
|
+
logger.trace("Acknowledged replication announcement repair exhausted for %s", hash);
|
|
4168
|
+
}
|
|
4169
|
+
}
|
|
4170
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
4171
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
4172
|
+
return;
|
|
4173
|
+
}
|
|
4174
|
+
if ([...this._replicationAnnouncementRepairTargets.values()].some((target) => !target.done)) {
|
|
4175
|
+
void this.replicationAnnouncementRepairDebounced?.call();
|
|
4176
|
+
return;
|
|
4177
|
+
}
|
|
4178
|
+
this._replicationAnnouncementRepairPending = false;
|
|
4179
|
+
this._replicationAnnouncementRepairTargets.clear();
|
|
4180
|
+
}
|
|
3692
4181
|
cancelCurrentReplicationStateAnnouncementRetry() {
|
|
3693
4182
|
this._replicationAnnouncementRetryGeneration += 1;
|
|
3694
4183
|
this._replicationAnnouncementRetryPending = false;
|
|
3695
4184
|
this._replicationAnnouncementRetryController?.abort();
|
|
3696
4185
|
this.replicationAnnouncementRetryDebounced?.close();
|
|
4186
|
+
this.cancelCurrentReplicationStateAnnouncementRepair();
|
|
3697
4187
|
}
|
|
3698
4188
|
async sendReplicationAnnouncement(message) {
|
|
3699
4189
|
// Advance before every post-mutation send, including successful ones. An
|
|
@@ -3701,10 +4191,12 @@ let SharedLog = (() => {
|
|
|
3701
4191
|
// local state; the generation mismatch forces one more current snapshot
|
|
3702
4192
|
// after that stale send settles.
|
|
3703
4193
|
this._replicationAnnouncementRetryGeneration += 1;
|
|
4194
|
+
this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
|
|
3704
4195
|
try {
|
|
3705
4196
|
await this.rpc.send(message, {
|
|
3706
4197
|
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
3707
4198
|
});
|
|
4199
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
3708
4200
|
}
|
|
3709
4201
|
catch (error) {
|
|
3710
4202
|
// The local replication-index mutation precedes all calls to this
|
|
@@ -3733,6 +4225,7 @@ let SharedLog = (() => {
|
|
|
3733
4225
|
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
3734
4226
|
signal: controller.signal,
|
|
3735
4227
|
});
|
|
4228
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
3736
4229
|
}
|
|
3737
4230
|
catch (error) {
|
|
3738
4231
|
if (this.closed ||
|
|
@@ -4139,61 +4632,135 @@ let SharedLog = (() => {
|
|
|
4139
4632
|
}
|
|
4140
4633
|
async removeReplicator(key, options) {
|
|
4141
4634
|
const keyHash = typeof key === "string" ? key : key.hashcode();
|
|
4142
|
-
const
|
|
4143
|
-
.
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
.
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4150
|
-
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
const isMe = this.node.identity.publicKey.hashcode() === keyHash;
|
|
4155
|
-
if (isMe) {
|
|
4156
|
-
// announce that we are no longer replicating
|
|
4157
|
-
await this.sendReplicationAnnouncement(new AllReplicatingSegmentsMessage({ segments: [] }));
|
|
4158
|
-
}
|
|
4159
|
-
if (options?.noEvent !== true) {
|
|
4160
|
-
const publicKey = toLocalPublicSignKey(key);
|
|
4161
|
-
if (publicKey) {
|
|
4162
|
-
this.events.dispatchEvent(new CustomEvent("replication:change", {
|
|
4163
|
-
detail: { publicKey },
|
|
4164
|
-
}));
|
|
4635
|
+
const expectedJoinWarmupGeneration = options?.expectedJoinWarmupGeneration !== undefined
|
|
4636
|
+
? options.expectedJoinWarmupGeneration
|
|
4637
|
+
: (this._joinWarmupGenerationByTarget.get(keyHash) ?? null);
|
|
4638
|
+
const ownsSubscriptionEpoch = () => options?.subscriptionEpoch === undefined ||
|
|
4639
|
+
this.isCurrentSubscriptionEpoch(keyHash, options.subscriptionEpoch);
|
|
4640
|
+
const ownsReplicationLifecycle = () => options?.replicationLifecycleController === undefined ||
|
|
4641
|
+
this.isReplicationLifecycleActive(options.replicationLifecycleController);
|
|
4642
|
+
const cancelExpectedJoinWarmupTarget = () => {
|
|
4643
|
+
if (expectedJoinWarmupGeneration !== null &&
|
|
4644
|
+
this._joinWarmupGenerationByTarget.get(keyHash) ===
|
|
4645
|
+
expectedJoinWarmupGeneration) {
|
|
4646
|
+
this.cancelJoinWarmupTarget(keyHash);
|
|
4165
4647
|
}
|
|
4166
|
-
|
|
4167
|
-
|
|
4648
|
+
};
|
|
4649
|
+
const isMe = this.node.identity.publicKey.hashcode() === keyHash;
|
|
4650
|
+
let receiveAdmissionBlocked = false;
|
|
4651
|
+
const blockAndDrainPeerReceives = async () => {
|
|
4652
|
+
if (!receiveAdmissionBlocked) {
|
|
4653
|
+
this.blockPeerReceiveAdmission(keyHash);
|
|
4654
|
+
receiveAdmissionBlocked = true;
|
|
4168
4655
|
}
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4656
|
+
await this.drainPeerReceiveHandlers(keyHash);
|
|
4657
|
+
};
|
|
4658
|
+
const cleanupDisconnectedPeer = async () => {
|
|
4659
|
+
await blockAndDrainPeerReceives();
|
|
4660
|
+
this.removePeerFromGidPeerHistory(keyHash);
|
|
4661
|
+
this.cleanupPeerDisconnectTracking(keyHash);
|
|
4662
|
+
this.removeRepairFrontierTarget(keyHash, {
|
|
4663
|
+
expectedJoinWarmupGeneration,
|
|
4176
4664
|
});
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
for (const [_k, v] of pendingMaturity) {
|
|
4181
|
-
clearTimeout(v.timeout);
|
|
4665
|
+
this._recentRepairDispatch.delete(keyHash);
|
|
4666
|
+
if (!isMe) {
|
|
4667
|
+
await this.syncronizer.onPeerDisconnected(keyHash);
|
|
4182
4668
|
}
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
//
|
|
4186
|
-
//
|
|
4187
|
-
|
|
4188
|
-
this.
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
|
|
4669
|
+
};
|
|
4670
|
+
let removed = false;
|
|
4671
|
+
// Replication-info updates already serialize per peer. Put the hash-wide
|
|
4672
|
+
// removal on the same queue so a newer reset cannot be deleted underneath
|
|
4673
|
+
// itself by an older unsubscribe callback.
|
|
4674
|
+
await this.withReplicationInfoApplyQueue(keyHash, async () => {
|
|
4675
|
+
if (!ownsReplicationLifecycle()) {
|
|
4676
|
+
return;
|
|
4677
|
+
}
|
|
4678
|
+
if (!ownsSubscriptionEpoch()) {
|
|
4679
|
+
// A reconnect may supersede an unsubscribe before its destructive
|
|
4680
|
+
// removal starts. Still retire the old connection's sync/request state
|
|
4681
|
+
// in lane order so the reconnect barrier cannot inherit stale caches.
|
|
4682
|
+
if (options?.cleanupIfSubscriptionSuperseded) {
|
|
4683
|
+
await cleanupDisconnectedPeer();
|
|
4684
|
+
}
|
|
4685
|
+
return;
|
|
4686
|
+
}
|
|
4687
|
+
if (options?.shouldRemove && !options.shouldRemove()) {
|
|
4688
|
+
return;
|
|
4689
|
+
}
|
|
4690
|
+
const wasReplicator = this.uniqueReplicators.has(keyHash);
|
|
4691
|
+
const deleted = await this.replicationIndex
|
|
4692
|
+
.iterate({
|
|
4693
|
+
query: { hash: keyHash },
|
|
4694
|
+
})
|
|
4695
|
+
.all();
|
|
4696
|
+
// Liveness evidence can arrive without a subscription transition while
|
|
4697
|
+
// this scan is pending. Stop new receives, drain those already admitted,
|
|
4698
|
+
// then revalidate immediately before the first destructive mutation.
|
|
4699
|
+
await blockAndDrainPeerReceives();
|
|
4700
|
+
if (options?.shouldRemove && !options.shouldRemove()) {
|
|
4701
|
+
return;
|
|
4702
|
+
}
|
|
4703
|
+
// Liveness removal must not cancel a still-current warmup until fresh
|
|
4704
|
+
// activity has been rechecked at the destructive boundary.
|
|
4705
|
+
cancelExpectedJoinWarmupTarget();
|
|
4706
|
+
await this.replicationIndex.del({ query: { hash: keyHash } });
|
|
4707
|
+
for (const result of deleted) {
|
|
4708
|
+
this.deleteNativeReplicationRange(result.value);
|
|
4709
|
+
}
|
|
4710
|
+
// Once the range indexes have removed the peer, make membership agree
|
|
4711
|
+
// before any later awaited bookkeeping can fail.
|
|
4712
|
+
this.uniqueReplicators.delete(keyHash);
|
|
4713
|
+
this._replicatorJoinEmitted.delete(keyHash);
|
|
4714
|
+
await this.updateOldestTimestampFromIndex();
|
|
4715
|
+
if (isMe) {
|
|
4716
|
+
// announce that we are no longer replicating
|
|
4717
|
+
await this.sendReplicationAnnouncement(new AllReplicatingSegmentsMessage({ segments: [] }));
|
|
4718
|
+
}
|
|
4719
|
+
if (options?.noEvent !== true) {
|
|
4720
|
+
const publicKey = toLocalPublicSignKey(key);
|
|
4721
|
+
if (publicKey) {
|
|
4722
|
+
this.events.dispatchEvent(new CustomEvent("replication:change", {
|
|
4723
|
+
detail: { publicKey },
|
|
4724
|
+
}));
|
|
4725
|
+
}
|
|
4726
|
+
else {
|
|
4727
|
+
throw new Error("Key was not a PublicSignKey");
|
|
4728
|
+
}
|
|
4729
|
+
}
|
|
4730
|
+
const timestamp = BigInt(+new Date());
|
|
4731
|
+
for (const x of deleted) {
|
|
4732
|
+
this.replicationChangeDebounceFn.add({
|
|
4733
|
+
range: x.value,
|
|
4734
|
+
type: "removed",
|
|
4735
|
+
timestamp,
|
|
4736
|
+
});
|
|
4737
|
+
}
|
|
4738
|
+
const pendingMaturity = this.pendingMaturity.get(keyHash);
|
|
4739
|
+
if (pendingMaturity) {
|
|
4740
|
+
for (const [_k, v] of pendingMaturity) {
|
|
4741
|
+
clearTimeout(v.timeout);
|
|
4742
|
+
}
|
|
4743
|
+
this.pendingMaturity.delete(keyHash);
|
|
4744
|
+
}
|
|
4745
|
+
// Keep local sync/prune state consistent even when a peer disappears
|
|
4746
|
+
// through replication-info updates without a topic unsubscribe event.
|
|
4747
|
+
await cleanupDisconnectedPeer();
|
|
4748
|
+
if (!isMe) {
|
|
4749
|
+
// Replication-info handlers release their receive lease before joining
|
|
4750
|
+
// this lane. Fence every handler queued behind this successful removal,
|
|
4751
|
+
// regardless of whether it came from liveness, startup pruning, or an
|
|
4752
|
+
// unsubscribe transition.
|
|
4753
|
+
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
4754
|
+
this.rebalanceParticipationDebounced?.call();
|
|
4755
|
+
}
|
|
4756
|
+
removed = true;
|
|
4757
|
+
options?.onRemoved?.({ wasReplicator });
|
|
4758
|
+
}).finally(() => {
|
|
4759
|
+
if (receiveAdmissionBlocked) {
|
|
4760
|
+
this.unblockPeerReceiveAdmission(keyHash);
|
|
4761
|
+
}
|
|
4762
|
+
});
|
|
4763
|
+
return removed;
|
|
4197
4764
|
}
|
|
4198
4765
|
async updateOldestTimestampFromIndex() {
|
|
4199
4766
|
const iterator = await this.replicationIndex.iterate({
|
|
@@ -4257,6 +4824,10 @@ let SharedLog = (() => {
|
|
|
4257
4824
|
if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
|
|
4258
4825
|
return undefined;
|
|
4259
4826
|
}
|
|
4827
|
+
// Preserve what the peer announced before duplicate filtering can empty the
|
|
4828
|
+
// working array. A repeated authoritative/non-empty announcement is still
|
|
4829
|
+
// proof of live membership after this process reopens.
|
|
4830
|
+
const announcedReplication = ranges.length > 0;
|
|
4260
4831
|
let isNewReplicator = false;
|
|
4261
4832
|
let timestamp = BigInt(ts ?? +new Date());
|
|
4262
4833
|
rebalance = rebalance == null ? true : rebalance;
|
|
@@ -4376,14 +4947,14 @@ let SharedLog = (() => {
|
|
|
4376
4947
|
// means "nothing new", not "stopped". Removing the peer here would let
|
|
4377
4948
|
// uniqueReplicators diverge from replicationIndex while the peer's
|
|
4378
4949
|
// ranges stay indexed.
|
|
4379
|
-
const announcedStopped = reset === true &&
|
|
4950
|
+
const announcedStopped = reset === true && !announcedReplication;
|
|
4380
4951
|
const stoppedTransition = announcedStopped
|
|
4381
4952
|
? this.uniqueReplicators.delete(fromHash)
|
|
4382
4953
|
: false;
|
|
4383
4954
|
if (announcedStopped) {
|
|
4384
4955
|
this._replicatorJoinEmitted.delete(fromHash);
|
|
4385
4956
|
}
|
|
4386
|
-
else if (
|
|
4957
|
+
else if (announcedReplication) {
|
|
4387
4958
|
this.uniqueReplicators.add(fromHash);
|
|
4388
4959
|
}
|
|
4389
4960
|
let now = +new Date();
|
|
@@ -4488,6 +5059,14 @@ let SharedLog = (() => {
|
|
|
4488
5059
|
this.rebalanceParticipationDebounced?.call();
|
|
4489
5060
|
}
|
|
4490
5061
|
}
|
|
5062
|
+
if (announcedReplication &&
|
|
5063
|
+
!from.equals(this.node.identity.publicKey) &&
|
|
5064
|
+
!this._replicatorJoinEmitted.has(fromHash)) {
|
|
5065
|
+
this._replicatorJoinEmitted.add(fromHash);
|
|
5066
|
+
this.events.dispatchEvent(new CustomEvent("replicator:join", {
|
|
5067
|
+
detail: { publicKey: from },
|
|
5068
|
+
}));
|
|
5069
|
+
}
|
|
4491
5070
|
return diffs;
|
|
4492
5071
|
}
|
|
4493
5072
|
async startAnnounceReplicating(range, options = {}) {
|
|
@@ -4679,16 +5258,40 @@ let SharedLog = (() => {
|
|
|
4679
5258
|
const observedAt = this._entryKnownPeerObservedAt.get(hash)?.get(peer);
|
|
4680
5259
|
return observedAt != null && Date.now() - observedAt <= maxAgeMs;
|
|
4681
5260
|
}
|
|
4682
|
-
markRepairSweepOptimisticPeer(gid, peer) {
|
|
5261
|
+
markRepairSweepOptimisticPeer(gid, peer, generation) {
|
|
4683
5262
|
let peers = this._repairSweepOptimisticGidPeersPending.get(gid);
|
|
4684
5263
|
if (!peers) {
|
|
4685
5264
|
peers = new Map();
|
|
4686
5265
|
this._repairSweepOptimisticGidPeersPending.set(gid, peers);
|
|
4687
5266
|
}
|
|
4688
|
-
|
|
5267
|
+
const current = peers.get(peer);
|
|
5268
|
+
peers.set(peer, {
|
|
5269
|
+
count: current?.generation === generation ? current.count + 1 : 1,
|
|
5270
|
+
generation,
|
|
5271
|
+
});
|
|
5272
|
+
let gids = this._repairSweepOptimisticGidsByPeer.get(peer);
|
|
5273
|
+
if (!gids) {
|
|
5274
|
+
gids = new Set();
|
|
5275
|
+
this._repairSweepOptimisticGidsByPeer.set(peer, gids);
|
|
5276
|
+
}
|
|
5277
|
+
gids.add(gid);
|
|
4689
5278
|
}
|
|
4690
5279
|
hasPendingRepairSweepOptimisticPeer(gid, peer) {
|
|
4691
|
-
return ((this._repairSweepOptimisticGidPeersPending.get(gid)?.get(peer) ||
|
|
5280
|
+
return ((this._repairSweepOptimisticGidPeersPending.get(gid)?.get(peer)?.count ||
|
|
5281
|
+
0) > 0);
|
|
5282
|
+
}
|
|
5283
|
+
clearRepairSweepOptimisticPeer(peer) {
|
|
5284
|
+
for (const gid of this._repairSweepOptimisticGidsByPeer.get(peer) ?? []) {
|
|
5285
|
+
const peers = this._repairSweepOptimisticGidPeersPending.get(gid);
|
|
5286
|
+
if (!peers) {
|
|
5287
|
+
continue;
|
|
5288
|
+
}
|
|
5289
|
+
peers.delete(peer);
|
|
5290
|
+
if (peers.size === 0) {
|
|
5291
|
+
this._repairSweepOptimisticGidPeersPending.delete(gid);
|
|
5292
|
+
}
|
|
5293
|
+
}
|
|
5294
|
+
this._repairSweepOptimisticGidsByPeer.delete(peer);
|
|
4692
5295
|
}
|
|
4693
5296
|
createEntryReplicatedForRepair(properties) {
|
|
4694
5297
|
const assignedToRangeBoundary = shouldAssignToRangeBoundary(properties.leaders, properties.replicas);
|
|
@@ -4757,14 +5360,221 @@ let SharedLog = (() => {
|
|
|
4757
5360
|
if (!pending) {
|
|
4758
5361
|
continue;
|
|
4759
5362
|
}
|
|
4760
|
-
for (const hash of hashList) {
|
|
4761
|
-
pending.delete(hash);
|
|
5363
|
+
for (const hash of hashList) {
|
|
5364
|
+
pending.delete(hash);
|
|
5365
|
+
}
|
|
5366
|
+
if (pending.size === 0) {
|
|
5367
|
+
this._repairFrontierByMode.get(mode)?.delete(target);
|
|
5368
|
+
this._repairFrontierBypassKnownPeersByMode.get(mode)?.delete(target);
|
|
5369
|
+
}
|
|
5370
|
+
}
|
|
5371
|
+
}
|
|
5372
|
+
getJoinWarmupGeneration(target) {
|
|
5373
|
+
let generation = this._joinWarmupGenerationByTarget.get(target);
|
|
5374
|
+
if (!generation) {
|
|
5375
|
+
generation = {};
|
|
5376
|
+
this._joinWarmupGenerationByTarget.set(target, generation);
|
|
5377
|
+
}
|
|
5378
|
+
return generation;
|
|
5379
|
+
}
|
|
5380
|
+
trackJoinWarmupTimer(target, timer) {
|
|
5381
|
+
let timers = this._joinWarmupRetryTimersByTarget.get(target);
|
|
5382
|
+
if (!timers) {
|
|
5383
|
+
timers = new Set();
|
|
5384
|
+
this._joinWarmupRetryTimersByTarget.set(target, timers);
|
|
5385
|
+
}
|
|
5386
|
+
timers.add(timer);
|
|
5387
|
+
this._repairRetryTimers.add(timer.handle);
|
|
5388
|
+
}
|
|
5389
|
+
untrackJoinWarmupTimer(target, timer) {
|
|
5390
|
+
this._repairRetryTimers.delete(timer.handle);
|
|
5391
|
+
const timers = this._joinWarmupRetryTimersByTarget.get(target);
|
|
5392
|
+
if (!timers) {
|
|
5393
|
+
return;
|
|
5394
|
+
}
|
|
5395
|
+
timers.delete(timer);
|
|
5396
|
+
if (timers.size === 0) {
|
|
5397
|
+
this._joinWarmupRetryTimersByTarget.delete(target);
|
|
5398
|
+
}
|
|
5399
|
+
}
|
|
5400
|
+
cancelJoinWarmupTimers(target) {
|
|
5401
|
+
const timers = this._joinWarmupRetryTimersByTarget.get(target);
|
|
5402
|
+
if (!timers) {
|
|
5403
|
+
return;
|
|
5404
|
+
}
|
|
5405
|
+
for (const timer of [...timers]) {
|
|
5406
|
+
clearTimeout(timer.handle);
|
|
5407
|
+
timer.resolve?.();
|
|
5408
|
+
this.untrackJoinWarmupTimer(target, timer);
|
|
5409
|
+
}
|
|
5410
|
+
}
|
|
5411
|
+
cancelJoinWarmupTarget(target) {
|
|
5412
|
+
this._joinWarmupGenerationByTarget.delete(target);
|
|
5413
|
+
const pendingWarmupPeers = this._repairSweepPendingPeersByMode.get("join-warmup");
|
|
5414
|
+
pendingWarmupPeers?.delete(target);
|
|
5415
|
+
if (pendingWarmupPeers?.size === 0) {
|
|
5416
|
+
this._repairSweepPendingModes.delete("join-warmup");
|
|
5417
|
+
}
|
|
5418
|
+
this._repairSweepJoinWarmupGenerationByTarget.delete(target);
|
|
5419
|
+
this.clearRepairSweepOptimisticPeer(target);
|
|
5420
|
+
this.cancelJoinWarmupTimers(target);
|
|
5421
|
+
this._joinWarmupScheduledRetriesByTarget.delete(target);
|
|
5422
|
+
const state = this._joinWarmupSendStateByTarget.get(target);
|
|
5423
|
+
if (!state) {
|
|
5424
|
+
return;
|
|
5425
|
+
}
|
|
5426
|
+
state.bypassKnownPeerHints = false;
|
|
5427
|
+
state.entries.clear();
|
|
5428
|
+
state.pending = false;
|
|
5429
|
+
if (!state.running) {
|
|
5430
|
+
this._joinWarmupSendStateByTarget.delete(target);
|
|
5431
|
+
}
|
|
5432
|
+
}
|
|
5433
|
+
cancelAllJoinWarmupTargets() {
|
|
5434
|
+
const targets = new Set([
|
|
5435
|
+
...this._joinWarmupGenerationByTarget.keys(),
|
|
5436
|
+
...this._joinWarmupRetryTimersByTarget.keys(),
|
|
5437
|
+
...this._joinWarmupScheduledRetriesByTarget.keys(),
|
|
5438
|
+
...this._joinWarmupSendStateByTarget.keys(),
|
|
5439
|
+
]);
|
|
5440
|
+
for (const target of targets) {
|
|
5441
|
+
this.cancelJoinWarmupTarget(target);
|
|
5442
|
+
}
|
|
5443
|
+
}
|
|
5444
|
+
async sleepJoinWarmupTracked(target, delayMs) {
|
|
5445
|
+
if (delayMs <= 0) {
|
|
5446
|
+
return;
|
|
5447
|
+
}
|
|
5448
|
+
await new Promise((resolve) => {
|
|
5449
|
+
let settled = false;
|
|
5450
|
+
let trackedTimer;
|
|
5451
|
+
const settle = () => {
|
|
5452
|
+
if (settled) {
|
|
5453
|
+
return;
|
|
5454
|
+
}
|
|
5455
|
+
settled = true;
|
|
5456
|
+
this.untrackJoinWarmupTimer(target, trackedTimer);
|
|
5457
|
+
resolve();
|
|
5458
|
+
};
|
|
5459
|
+
const handle = setTimeout(settle, delayMs);
|
|
5460
|
+
handle.unref?.();
|
|
5461
|
+
trackedTimer = { handle, resolve: settle };
|
|
5462
|
+
this.trackJoinWarmupTimer(target, trackedTimer);
|
|
5463
|
+
});
|
|
5464
|
+
}
|
|
5465
|
+
scheduleJoinWarmupRetries(target, generation, delaysMs, entries, bypassKnownPeerHints) {
|
|
5466
|
+
if (this.closed ||
|
|
5467
|
+
this._joinWarmupGenerationByTarget.get(target) !== generation) {
|
|
5468
|
+
return;
|
|
5469
|
+
}
|
|
5470
|
+
let scheduled = this._joinWarmupScheduledRetriesByTarget.get(target);
|
|
5471
|
+
if (scheduled?.generation !== generation) {
|
|
5472
|
+
this.cancelJoinWarmupTimers(target);
|
|
5473
|
+
this._joinWarmupScheduledRetriesByTarget.delete(target);
|
|
5474
|
+
scheduled = undefined;
|
|
5475
|
+
}
|
|
5476
|
+
if (!scheduled) {
|
|
5477
|
+
scheduled = {
|
|
5478
|
+
generation,
|
|
5479
|
+
slotsByDelay: new Map(),
|
|
5480
|
+
};
|
|
5481
|
+
this._joinWarmupScheduledRetriesByTarget.set(target, scheduled);
|
|
5482
|
+
}
|
|
5483
|
+
const delays = [...new Set(delaysMs)];
|
|
5484
|
+
const batch = {
|
|
5485
|
+
bypassKnownPeerHints,
|
|
5486
|
+
entries: new Map(entries),
|
|
5487
|
+
remainingAttempts: delays.length,
|
|
5488
|
+
};
|
|
5489
|
+
const now = Date.now();
|
|
5490
|
+
for (const delayMs of delays) {
|
|
5491
|
+
let slot = scheduled.slotsByDelay.get(delayMs);
|
|
5492
|
+
if (!slot) {
|
|
5493
|
+
slot = { cohorts: [], head: 0 };
|
|
5494
|
+
scheduled.slotsByDelay.set(delayMs, slot);
|
|
5495
|
+
}
|
|
5496
|
+
const tail = slot.cohorts.at(-1);
|
|
5497
|
+
const dueAt = Math.max(tail?.dueAt ?? 0, now + delayMs);
|
|
5498
|
+
if (tail?.dueAt === dueAt) {
|
|
5499
|
+
tail.batches.push(batch);
|
|
5500
|
+
}
|
|
5501
|
+
else {
|
|
5502
|
+
slot.cohorts.push({
|
|
5503
|
+
batches: [batch],
|
|
5504
|
+
dueAt,
|
|
5505
|
+
});
|
|
5506
|
+
}
|
|
5507
|
+
this.armJoinWarmupRetrySlot(target, scheduled, delayMs, slot);
|
|
5508
|
+
}
|
|
5509
|
+
}
|
|
5510
|
+
armJoinWarmupRetrySlot(target, scheduled, delayMs, slot) {
|
|
5511
|
+
const nextDueAt = slot.cohorts[slot.head]?.dueAt;
|
|
5512
|
+
if (nextDueAt == null) {
|
|
5513
|
+
return;
|
|
5514
|
+
}
|
|
5515
|
+
if (slot.timer && slot.timerDueAt === nextDueAt) {
|
|
5516
|
+
return;
|
|
5517
|
+
}
|
|
5518
|
+
if (slot.timer) {
|
|
5519
|
+
clearTimeout(slot.timer.handle);
|
|
5520
|
+
this.untrackJoinWarmupTimer(target, slot.timer);
|
|
5521
|
+
}
|
|
5522
|
+
let trackedTimer;
|
|
5523
|
+
const handle = setTimeout(() => {
|
|
5524
|
+
this.untrackJoinWarmupTimer(target, trackedTimer);
|
|
5525
|
+
if (slot.timer !== trackedTimer) {
|
|
5526
|
+
return;
|
|
5527
|
+
}
|
|
5528
|
+
slot.timer = undefined;
|
|
5529
|
+
slot.timerDueAt = undefined;
|
|
5530
|
+
const current = this._joinWarmupScheduledRetriesByTarget.get(target);
|
|
5531
|
+
if (current !== scheduled ||
|
|
5532
|
+
current.slotsByDelay.get(delayMs) !== slot) {
|
|
5533
|
+
return;
|
|
5534
|
+
}
|
|
5535
|
+
const dueEntries = new Map();
|
|
5536
|
+
let bypassKnownPeerHints = false;
|
|
5537
|
+
const now = Date.now();
|
|
5538
|
+
while (slot.head < slot.cohorts.length &&
|
|
5539
|
+
slot.cohorts[slot.head].dueAt <= now) {
|
|
5540
|
+
const cohort = slot.cohorts[slot.head++];
|
|
5541
|
+
for (const batch of cohort.batches) {
|
|
5542
|
+
for (const [hash, entry] of batch.entries) {
|
|
5543
|
+
dueEntries.set(hash, entry);
|
|
5544
|
+
}
|
|
5545
|
+
bypassKnownPeerHints ||=
|
|
5546
|
+
batch.bypassKnownPeerHints;
|
|
5547
|
+
batch.remainingAttempts -= 1;
|
|
5548
|
+
if (batch.remainingAttempts === 0) {
|
|
5549
|
+
batch.entries.clear();
|
|
5550
|
+
}
|
|
5551
|
+
}
|
|
5552
|
+
cohort.batches.length = 0;
|
|
5553
|
+
}
|
|
5554
|
+
if (dueEntries.size > 0 &&
|
|
5555
|
+
!this.closed &&
|
|
5556
|
+
this._joinWarmupGenerationByTarget.get(target) ===
|
|
5557
|
+
scheduled.generation) {
|
|
5558
|
+
this.queueJoinWarmupSend(target, scheduled.generation, dueEntries, bypassKnownPeerHints);
|
|
4762
5559
|
}
|
|
4763
|
-
if (
|
|
4764
|
-
|
|
4765
|
-
|
|
5560
|
+
if (slot.head === slot.cohorts.length) {
|
|
5561
|
+
current.slotsByDelay.delete(delayMs);
|
|
5562
|
+
if (current.slotsByDelay.size === 0) {
|
|
5563
|
+
this._joinWarmupScheduledRetriesByTarget.delete(target);
|
|
5564
|
+
}
|
|
5565
|
+
return;
|
|
4766
5566
|
}
|
|
4767
|
-
|
|
5567
|
+
if (slot.head >= 1_024 && slot.head * 2 >= slot.cohorts.length) {
|
|
5568
|
+
slot.cohorts = slot.cohorts.slice(slot.head);
|
|
5569
|
+
slot.head = 0;
|
|
5570
|
+
}
|
|
5571
|
+
this.armJoinWarmupRetrySlot(target, current, delayMs, slot);
|
|
5572
|
+
}, Math.max(0, nextDueAt - Date.now()));
|
|
5573
|
+
handle.unref?.();
|
|
5574
|
+
trackedTimer = { handle };
|
|
5575
|
+
slot.timer = trackedTimer;
|
|
5576
|
+
slot.timerDueAt = nextDueAt;
|
|
5577
|
+
this.trackJoinWarmupTimer(target, trackedTimer);
|
|
4768
5578
|
}
|
|
4769
5579
|
async getFullReplicaRepairCandidates(extraPeers, options) {
|
|
4770
5580
|
const candidates = new Set([
|
|
@@ -4795,7 +5605,13 @@ let SharedLog = (() => {
|
|
|
4795
5605
|
}
|
|
4796
5606
|
return candidates;
|
|
4797
5607
|
}
|
|
4798
|
-
removeRepairFrontierTarget(target) {
|
|
5608
|
+
removeRepairFrontierTarget(target, options) {
|
|
5609
|
+
if (options?.expectedJoinWarmupGeneration === undefined ||
|
|
5610
|
+
(options.expectedJoinWarmupGeneration !== null &&
|
|
5611
|
+
this._joinWarmupGenerationByTarget.get(target) ===
|
|
5612
|
+
options.expectedJoinWarmupGeneration)) {
|
|
5613
|
+
this.cancelJoinWarmupTarget(target);
|
|
5614
|
+
}
|
|
4799
5615
|
for (const mode of REPAIR_DISPATCH_MODES) {
|
|
4800
5616
|
this._repairFrontierByMode.get(mode)?.delete(target);
|
|
4801
5617
|
this._repairFrontierActiveTargetsByMode.get(mode)?.delete(target);
|
|
@@ -4870,6 +5686,89 @@ let SharedLog = (() => {
|
|
|
4870
5686
|
targets: [target],
|
|
4871
5687
|
});
|
|
4872
5688
|
}
|
|
5689
|
+
queueJoinWarmupSend(target, generation, entries, bypassKnownPeerHints) {
|
|
5690
|
+
if (this.closed ||
|
|
5691
|
+
this._joinWarmupGenerationByTarget.get(target) !== generation) {
|
|
5692
|
+
return;
|
|
5693
|
+
}
|
|
5694
|
+
let state = this._joinWarmupSendStateByTarget.get(target);
|
|
5695
|
+
if (!state) {
|
|
5696
|
+
state = {
|
|
5697
|
+
bypassKnownPeerHints: false,
|
|
5698
|
+
entries: new Map(),
|
|
5699
|
+
generation,
|
|
5700
|
+
lastCompletedAt: Number.NEGATIVE_INFINITY,
|
|
5701
|
+
pending: false,
|
|
5702
|
+
running: false,
|
|
5703
|
+
};
|
|
5704
|
+
this._joinWarmupSendStateByTarget.set(target, state);
|
|
5705
|
+
}
|
|
5706
|
+
else if (state.generation !== generation) {
|
|
5707
|
+
state.bypassKnownPeerHints = false;
|
|
5708
|
+
state.entries.clear();
|
|
5709
|
+
state.pending = false;
|
|
5710
|
+
}
|
|
5711
|
+
for (const [hash, entry] of entries) {
|
|
5712
|
+
state.entries.set(hash, entry);
|
|
5713
|
+
}
|
|
5714
|
+
state.bypassKnownPeerHints ||= bypassKnownPeerHints;
|
|
5715
|
+
state.generation = generation;
|
|
5716
|
+
state.pending = true;
|
|
5717
|
+
if (state.running) {
|
|
5718
|
+
return;
|
|
5719
|
+
}
|
|
5720
|
+
void this.drainJoinWarmupSends(target, state).catch((error) => logger.error(error));
|
|
5721
|
+
}
|
|
5722
|
+
async drainJoinWarmupSends(target, state) {
|
|
5723
|
+
if (state.running) {
|
|
5724
|
+
return;
|
|
5725
|
+
}
|
|
5726
|
+
state.running = true;
|
|
5727
|
+
try {
|
|
5728
|
+
while (state.pending) {
|
|
5729
|
+
state.pending = false;
|
|
5730
|
+
const generation = state.generation;
|
|
5731
|
+
const entries = new Map(state.entries);
|
|
5732
|
+
state.entries.clear();
|
|
5733
|
+
const bypassKnownPeerHints = state.bypassKnownPeerHints;
|
|
5734
|
+
state.bypassKnownPeerHints = false;
|
|
5735
|
+
const spacingMs = Math.max(0, state.lastCompletedAt + JOIN_WARMUP_SEND_SPACING_MS - Date.now());
|
|
5736
|
+
await this.sleepJoinWarmupTracked(target, spacingMs);
|
|
5737
|
+
if (this.closed ||
|
|
5738
|
+
state.generation !== generation ||
|
|
5739
|
+
this._joinWarmupGenerationByTarget.get(target) !== generation) {
|
|
5740
|
+
continue;
|
|
5741
|
+
}
|
|
5742
|
+
if (entries.size === 0) {
|
|
5743
|
+
continue;
|
|
5744
|
+
}
|
|
5745
|
+
this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
|
|
5746
|
+
try {
|
|
5747
|
+
await this.sendRepairEntriesWithTransport(target, entries, "simple", {
|
|
5748
|
+
bypassKnownPeers: bypassKnownPeerHints,
|
|
5749
|
+
bypassRecentKnownPeers: bypassKnownPeerHints,
|
|
5750
|
+
});
|
|
5751
|
+
}
|
|
5752
|
+
catch (error) {
|
|
5753
|
+
logger.error(error);
|
|
5754
|
+
}
|
|
5755
|
+
finally {
|
|
5756
|
+
state.lastCompletedAt = Date.now();
|
|
5757
|
+
}
|
|
5758
|
+
}
|
|
5759
|
+
}
|
|
5760
|
+
finally {
|
|
5761
|
+
state.running = false;
|
|
5762
|
+
if (this._joinWarmupSendStateByTarget.get(target) === state) {
|
|
5763
|
+
if (state.pending) {
|
|
5764
|
+
void this.drainJoinWarmupSends(target, state).catch((error) => logger.error(error));
|
|
5765
|
+
}
|
|
5766
|
+
else if (!this._joinWarmupGenerationByTarget.has(target)) {
|
|
5767
|
+
this._joinWarmupSendStateByTarget.delete(target);
|
|
5768
|
+
}
|
|
5769
|
+
}
|
|
5770
|
+
}
|
|
5771
|
+
}
|
|
4873
5772
|
async sendMaybeMissingEntriesNow(target, entries, options) {
|
|
4874
5773
|
if (entries.size === 0) {
|
|
4875
5774
|
return;
|
|
@@ -5074,25 +5973,41 @@ let SharedLog = (() => {
|
|
|
5074
5973
|
const bucket = this._repairMetrics[options.mode];
|
|
5075
5974
|
bucket.dispatches += 1;
|
|
5076
5975
|
bucket.entries += filteredEntries.size;
|
|
5976
|
+
const joinWarmupGeneration = options.mode === "join-warmup"
|
|
5977
|
+
? this.getJoinWarmupGeneration(target)
|
|
5978
|
+
: undefined;
|
|
5979
|
+
const bypassKnownPeerHints = this.shouldBypassKnownPeerHints(options.mode, options.bypassKnownPeerHints);
|
|
5077
5980
|
const run = (transport) => {
|
|
5078
|
-
if (transport === "simple"
|
|
5079
|
-
|
|
5981
|
+
if (transport === "simple" &&
|
|
5982
|
+
options.mode === "join-warmup" &&
|
|
5983
|
+
joinWarmupGeneration) {
|
|
5984
|
+
this.queueJoinWarmupSend(target, joinWarmupGeneration, filteredEntries, bypassKnownPeerHints);
|
|
5985
|
+
return;
|
|
5080
5986
|
}
|
|
5081
|
-
|
|
5987
|
+
if (transport === "rateless") {
|
|
5082
5988
|
bucket.ratelessFirstPasses += 1;
|
|
5083
5989
|
}
|
|
5084
|
-
|
|
5990
|
+
else {
|
|
5991
|
+
bucket.simpleFallbackPasses += 1;
|
|
5992
|
+
}
|
|
5085
5993
|
return Promise.resolve(this.sendRepairEntriesWithTransport(target, filteredEntries, transport, {
|
|
5086
5994
|
bypassKnownPeers: bypassKnownPeerHints,
|
|
5087
5995
|
bypassRecentKnownPeers: bypassKnownPeerHints,
|
|
5088
5996
|
})).catch((error) => logger.error(error));
|
|
5089
5997
|
};
|
|
5998
|
+
const delayedJoinWarmupRetries = [];
|
|
5090
5999
|
retrySchedule.forEach((delayMs, index) => {
|
|
5091
6000
|
const transport = getRepairTransportForAttempt(options.mode, index);
|
|
5092
6001
|
if (delayMs === 0) {
|
|
5093
6002
|
void run(transport);
|
|
5094
6003
|
return;
|
|
5095
6004
|
}
|
|
6005
|
+
if (options.mode === "join-warmup" &&
|
|
6006
|
+
joinWarmupGeneration &&
|
|
6007
|
+
transport === "simple") {
|
|
6008
|
+
delayedJoinWarmupRetries.push(delayMs);
|
|
6009
|
+
return;
|
|
6010
|
+
}
|
|
5096
6011
|
const timer = setTimeout(() => {
|
|
5097
6012
|
this._repairRetryTimers.delete(timer);
|
|
5098
6013
|
if (this.closed) {
|
|
@@ -5103,15 +6018,29 @@ let SharedLog = (() => {
|
|
|
5103
6018
|
timer.unref?.();
|
|
5104
6019
|
this._repairRetryTimers.add(timer);
|
|
5105
6020
|
});
|
|
6021
|
+
if (joinWarmupGeneration && delayedJoinWarmupRetries.length > 0) {
|
|
6022
|
+
this.scheduleJoinWarmupRetries(target, joinWarmupGeneration, delayedJoinWarmupRetries, filteredEntries, bypassKnownPeerHints);
|
|
6023
|
+
}
|
|
5106
6024
|
}
|
|
5107
6025
|
scheduleRepairSweep(options) {
|
|
5108
|
-
this._repairSweepPendingModes.add(options.mode);
|
|
5109
6026
|
const pendingPeers = this._repairSweepPendingPeersByMode.get(options.mode);
|
|
5110
6027
|
if (pendingPeers) {
|
|
5111
6028
|
for (const peer of options.peers ?? []) {
|
|
6029
|
+
if (options.mode === "join-warmup") {
|
|
6030
|
+
const generation = options.joinWarmupGenerations?.get(peer) ??
|
|
6031
|
+
this.getJoinWarmupGeneration(peer);
|
|
6032
|
+
if (this._joinWarmupGenerationByTarget.get(peer) !== generation) {
|
|
6033
|
+
continue;
|
|
6034
|
+
}
|
|
6035
|
+
this._repairSweepJoinWarmupGenerationByTarget.set(peer, generation);
|
|
6036
|
+
}
|
|
5112
6037
|
pendingPeers.add(peer);
|
|
5113
6038
|
}
|
|
5114
6039
|
}
|
|
6040
|
+
if (!pendingPeers || pendingPeers.size === 0) {
|
|
6041
|
+
return;
|
|
6042
|
+
}
|
|
6043
|
+
this._repairSweepPendingModes.add(options.mode);
|
|
5115
6044
|
if (!this._repairSweepRunning && !this.closed) {
|
|
5116
6045
|
this._repairSweepRunning = true;
|
|
5117
6046
|
void this.runRepairSweep();
|
|
@@ -5162,10 +6091,26 @@ let SharedLog = (() => {
|
|
|
5162
6091
|
while (!this.closed) {
|
|
5163
6092
|
const pendingModes = new Set(this._repairSweepPendingModes);
|
|
5164
6093
|
const pendingPeersByMode = cloneRepairPendingPeersByMode(this._repairSweepPendingPeersByMode);
|
|
6094
|
+
const pendingJoinWarmupGenerations = new Map(this._repairSweepJoinWarmupGenerationByTarget);
|
|
5165
6095
|
this._repairSweepPendingModes.clear();
|
|
5166
6096
|
for (const peers of this._repairSweepPendingPeersByMode.values()) {
|
|
5167
6097
|
peers.clear();
|
|
5168
6098
|
}
|
|
6099
|
+
this._repairSweepJoinWarmupGenerationByTarget.clear();
|
|
6100
|
+
const pendingJoinWarmupPeers = pendingPeersByMode.get("join-warmup");
|
|
6101
|
+
const pruneStaleJoinWarmupPeers = () => {
|
|
6102
|
+
for (const peer of [...(pendingJoinWarmupPeers ?? [])]) {
|
|
6103
|
+
if (this._joinWarmupGenerationByTarget.get(peer) !==
|
|
6104
|
+
pendingJoinWarmupGenerations.get(peer)) {
|
|
6105
|
+
pendingJoinWarmupPeers?.delete(peer);
|
|
6106
|
+
}
|
|
6107
|
+
}
|
|
6108
|
+
if (pendingJoinWarmupPeers?.size === 0) {
|
|
6109
|
+
pendingModes.delete("join-warmup");
|
|
6110
|
+
}
|
|
6111
|
+
return pendingModes.size > 0;
|
|
6112
|
+
};
|
|
6113
|
+
pruneStaleJoinWarmupPeers();
|
|
5169
6114
|
if (pendingModes.size === 0) {
|
|
5170
6115
|
return;
|
|
5171
6116
|
}
|
|
@@ -5182,14 +6127,14 @@ let SharedLog = (() => {
|
|
|
5182
6127
|
._repairSweepOptimisticGidPeersPending) {
|
|
5183
6128
|
let matchedPeers;
|
|
5184
6129
|
let matchedCounts;
|
|
5185
|
-
for (const [peer,
|
|
6130
|
+
for (const [peer, state] of peerCounts) {
|
|
5186
6131
|
if (!modePeers.has(peer)) {
|
|
5187
6132
|
continue;
|
|
5188
6133
|
}
|
|
5189
6134
|
matchedPeers ||= new Set();
|
|
5190
6135
|
matchedCounts ||= new Map();
|
|
5191
6136
|
matchedPeers.add(peer);
|
|
5192
|
-
matchedCounts.set(peer,
|
|
6137
|
+
matchedCounts.set(peer, { ...state });
|
|
5193
6138
|
}
|
|
5194
6139
|
if (matchedPeers && matchedCounts) {
|
|
5195
6140
|
optimisticGidPeers.set(gid, matchedPeers);
|
|
@@ -5211,6 +6156,7 @@ let SharedLog = (() => {
|
|
|
5211
6156
|
const fullReplicaRepairCandidates = await this.getFullReplicaRepairCandidates(pendingRepairPeers, {
|
|
5212
6157
|
includeSubscribers: false,
|
|
5213
6158
|
});
|
|
6159
|
+
pruneStaleJoinWarmupPeers();
|
|
5214
6160
|
const fullReplicaRepairCandidateCount = Math.max(1, fullReplicaRepairCandidates.size);
|
|
5215
6161
|
const nextFrontierByMode = new Map([
|
|
5216
6162
|
["join-authoritative", new Map()],
|
|
@@ -5222,6 +6168,16 @@ let SharedLog = (() => {
|
|
|
5222
6168
|
if (!entries || entries.size === 0) {
|
|
5223
6169
|
return;
|
|
5224
6170
|
}
|
|
6171
|
+
if (mode === "join-warmup" &&
|
|
6172
|
+
this._joinWarmupGenerationByTarget.get(target) !==
|
|
6173
|
+
pendingJoinWarmupGenerations.get(target)) {
|
|
6174
|
+
targets?.delete(target);
|
|
6175
|
+
pendingJoinWarmupPeers?.delete(target);
|
|
6176
|
+
if (pendingJoinWarmupPeers?.size === 0) {
|
|
6177
|
+
pendingModes.delete("join-warmup");
|
|
6178
|
+
}
|
|
6179
|
+
return;
|
|
6180
|
+
}
|
|
5225
6181
|
this.dispatchMaybeMissingEntries(target, entries, {
|
|
5226
6182
|
bypassRecentDedupe: true,
|
|
5227
6183
|
bypassKnownPeerHints: mode === "churn" ||
|
|
@@ -5233,6 +6189,15 @@ let SharedLog = (() => {
|
|
|
5233
6189
|
targets?.delete(target);
|
|
5234
6190
|
};
|
|
5235
6191
|
const queueEntryForTarget = (mode, target, entry) => {
|
|
6192
|
+
if (mode === "join-warmup" &&
|
|
6193
|
+
this._joinWarmupGenerationByTarget.get(target) !==
|
|
6194
|
+
pendingJoinWarmupGenerations.get(target)) {
|
|
6195
|
+
pendingJoinWarmupPeers?.delete(target);
|
|
6196
|
+
if (pendingJoinWarmupPeers?.size === 0) {
|
|
6197
|
+
pendingModes.delete("join-warmup");
|
|
6198
|
+
}
|
|
6199
|
+
return;
|
|
6200
|
+
}
|
|
5236
6201
|
const sweepTargets = nextFrontierByMode.get(mode);
|
|
5237
6202
|
if (sweepTargets) {
|
|
5238
6203
|
let sweepSet = sweepTargets.get(target);
|
|
@@ -5260,14 +6225,17 @@ let SharedLog = (() => {
|
|
|
5260
6225
|
if ((this._nativeBackbone ?? this._nativeSharedLogState) &&
|
|
5261
6226
|
residentEntriesByHash &&
|
|
5262
6227
|
!this.hasCustomFindLeaders()) {
|
|
5263
|
-
const repairDispatchPlan =
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5269
|
-
|
|
5270
|
-
|
|
6228
|
+
const repairDispatchPlan = pruneStaleJoinWarmupPeers()
|
|
6229
|
+
? await this.planResidentRepairDispatchBatch({
|
|
6230
|
+
pendingModes,
|
|
6231
|
+
pendingPeersByMode,
|
|
6232
|
+
optimisticGidPeersByMode,
|
|
6233
|
+
fullReplicaRepairCandidates,
|
|
6234
|
+
fullReplicaRepairCandidateCount,
|
|
6235
|
+
selfHash: this.node.identity.publicKey.hashcode(),
|
|
6236
|
+
})
|
|
6237
|
+
: new Map();
|
|
6238
|
+
pruneStaleJoinWarmupPeers();
|
|
5271
6239
|
for (const [mode, targets] of repairDispatchPlan) {
|
|
5272
6240
|
for (const [target, hashes] of targets) {
|
|
5273
6241
|
for (const hash of hashes) {
|
|
@@ -5279,11 +6247,16 @@ let SharedLog = (() => {
|
|
|
5279
6247
|
}
|
|
5280
6248
|
}
|
|
5281
6249
|
}
|
|
5282
|
-
else {
|
|
6250
|
+
else if (pruneStaleJoinWarmupPeers()) {
|
|
5283
6251
|
const iterator = this.entryCoordinatesIndex.iterate({});
|
|
5284
6252
|
try {
|
|
5285
|
-
while (!this.closed &&
|
|
6253
|
+
while (!this.closed &&
|
|
6254
|
+
!iterator.done() &&
|
|
6255
|
+
pruneStaleJoinWarmupPeers()) {
|
|
5286
6256
|
const entries = await iterator.next(REPAIR_SWEEP_ENTRY_BATCH_SIZE);
|
|
6257
|
+
if (!pruneStaleJoinWarmupPeers()) {
|
|
6258
|
+
break;
|
|
6259
|
+
}
|
|
5287
6260
|
const entryReplicatedBatch = entries.map((entry) => entry.value);
|
|
5288
6261
|
const requestedReplicasBatch = entryReplicatedBatch.map((entry) => decodeReplicas(entry).getValue(this));
|
|
5289
6262
|
const repairDispatchPlan = await this.planRepairDispatchBatch({
|
|
@@ -5296,6 +6269,9 @@ let SharedLog = (() => {
|
|
|
5296
6269
|
fullReplicaRepairCandidateCount,
|
|
5297
6270
|
selfHash: this.node.identity.publicKey.hashcode(),
|
|
5298
6271
|
});
|
|
6272
|
+
if (!pruneStaleJoinWarmupPeers()) {
|
|
6273
|
+
break;
|
|
6274
|
+
}
|
|
5299
6275
|
const entriesByHash = new Map(entryReplicatedBatch.map((entry) => [entry.hash, entry]));
|
|
5300
6276
|
for (const [mode, targets] of repairDispatchPlan) {
|
|
5301
6277
|
for (const [target, hashes] of targets) {
|
|
@@ -5319,14 +6295,26 @@ let SharedLog = (() => {
|
|
|
5319
6295
|
if (!pendingPeerCounts) {
|
|
5320
6296
|
continue;
|
|
5321
6297
|
}
|
|
5322
|
-
for (const [peer,
|
|
5323
|
-
const current = pendingPeerCounts.get(peer)
|
|
5324
|
-
|
|
6298
|
+
for (const [peer, consumed] of peerCounts) {
|
|
6299
|
+
const current = pendingPeerCounts.get(peer);
|
|
6300
|
+
if (!current ||
|
|
6301
|
+
current.generation !== consumed.generation) {
|
|
6302
|
+
continue;
|
|
6303
|
+
}
|
|
6304
|
+
const next = current.count - consumed.count;
|
|
5325
6305
|
if (next > 0) {
|
|
5326
|
-
pendingPeerCounts.set(peer,
|
|
6306
|
+
pendingPeerCounts.set(peer, {
|
|
6307
|
+
count: next,
|
|
6308
|
+
generation: current.generation,
|
|
6309
|
+
});
|
|
5327
6310
|
}
|
|
5328
6311
|
else {
|
|
5329
6312
|
pendingPeerCounts.delete(peer);
|
|
6313
|
+
const gids = this._repairSweepOptimisticGidsByPeer.get(peer);
|
|
6314
|
+
gids?.delete(gid);
|
|
6315
|
+
if (gids?.size === 0) {
|
|
6316
|
+
this._repairSweepOptimisticGidsByPeer.delete(peer);
|
|
6317
|
+
}
|
|
5330
6318
|
}
|
|
5331
6319
|
}
|
|
5332
6320
|
if (pendingPeerCounts.size === 0) {
|
|
@@ -5772,10 +6760,20 @@ let SharedLog = (() => {
|
|
|
5772
6760
|
if (toPrune.size === 0) {
|
|
5773
6761
|
return;
|
|
5774
6762
|
}
|
|
5775
|
-
|
|
6763
|
+
const pruneTasks = this.prune(toPrune);
|
|
6764
|
+
const confirmationTasks = [];
|
|
5776
6765
|
for (const hash of responseStillApplies) {
|
|
5777
|
-
|
|
6766
|
+
const pendingDelete = this._checkedPrune.getPendingDelete(hash);
|
|
6767
|
+
if (pendingDelete) {
|
|
6768
|
+
confirmationTasks.push(Promise.resolve(pendingDelete.resolve(publicKeyHash)));
|
|
6769
|
+
}
|
|
5778
6770
|
}
|
|
6771
|
+
// The restarted prune session owns its own bounded timeout and revalidates
|
|
6772
|
+
// before deletion. Only the peer-derived confirmations must remain inside
|
|
6773
|
+
// this receive lease; waiting for the whole prune session could stall close
|
|
6774
|
+
// or disconnect until its background timeout.
|
|
6775
|
+
void Promise.allSettled(pruneTasks);
|
|
6776
|
+
await Promise.allSettled(confirmationTasks);
|
|
5779
6777
|
}
|
|
5780
6778
|
async append(data, options) {
|
|
5781
6779
|
this.throwIfNativeDurableCommitFailed();
|
|
@@ -8193,20 +9191,37 @@ let SharedLog = (() => {
|
|
|
8193
9191
|
this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
|
|
8194
9192
|
this._checkedPrune = new CheckedPruneCoordinator();
|
|
8195
9193
|
this._pendingIHave = new Map();
|
|
9194
|
+
this._pendingIHaveCallbacks = new Set();
|
|
8196
9195
|
this.latestReplicationInfoMessage = new Map();
|
|
8197
9196
|
this._replicationInfoBlockedPeers = new Set();
|
|
8198
9197
|
this._replicationInfoRequestByPeer = new Map();
|
|
9198
|
+
// Terminal close/drop drains the previous lifecycle before another open can
|
|
9199
|
+
// install fresh lanes and opaque per-subscription ownership tokens.
|
|
8199
9200
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
9201
|
+
this._replicationInfoReceiveEpochByPeer = new Map();
|
|
9202
|
+
this._subscriptionEpochByPeer = new Map();
|
|
9203
|
+
this._pendingReplicatorLeaveByPeer = new Set();
|
|
9204
|
+
this._activeReceiveHandlersByPeer = new Map();
|
|
9205
|
+
this._receiveHandlerDrainByPeer = new Map();
|
|
9206
|
+
this._receiveCleanupGateByPeer = new Map();
|
|
9207
|
+
this._subscriptionOpeningEpochByPeer = new Map();
|
|
9208
|
+
this._openingSyncCapabilitiesByPeer = new Map();
|
|
8200
9209
|
this._repairRetryTimers = new Set();
|
|
8201
9210
|
this._recentRepairDispatch = new Map();
|
|
8202
9211
|
this._repairSweepRunning = false;
|
|
8203
9212
|
this._repairSweepPendingModes = new Set();
|
|
8204
9213
|
this._repairSweepPendingPeersByMode = createRepairPendingPeersByMode();
|
|
9214
|
+
this._repairSweepJoinWarmupGenerationByTarget = new Map();
|
|
8205
9215
|
this._repairFrontierByMode = createRepairFrontierByMode();
|
|
8206
9216
|
this._repairFrontierActiveTargetsByMode = createRepairActiveTargetsByMode();
|
|
8207
9217
|
this._repairFrontierBypassKnownPeersByMode =
|
|
8208
9218
|
createRepairFrontierBypassKnownPeersByMode();
|
|
9219
|
+
this._joinWarmupGenerationByTarget = new Map();
|
|
9220
|
+
this._joinWarmupSendStateByTarget ??= new Map();
|
|
9221
|
+
this._joinWarmupRetryTimersByTarget = new Map();
|
|
9222
|
+
this._joinWarmupScheduledRetriesByTarget = new Map();
|
|
8209
9223
|
this._repairSweepOptimisticGidPeersPending = new Map();
|
|
9224
|
+
this._repairSweepOptimisticGidsByPeer = new Map();
|
|
8210
9225
|
this._entryKnownPeers = new Map();
|
|
8211
9226
|
this._entryKnownPeerObservedAt = new Map();
|
|
8212
9227
|
this._joinAuthoritativeRepairTimersByDelay = new Map();
|
|
@@ -8273,6 +9288,7 @@ let SharedLog = (() => {
|
|
|
8273
9288
|
}
|
|
8274
9289
|
this._closeController = new AbortController();
|
|
8275
9290
|
this.setupReplicationAnnouncementRetryFunction();
|
|
9291
|
+
this.setupReplicationAnnouncementRepairFunction();
|
|
8276
9292
|
this._closeController.signal.addEventListener("abort", () => {
|
|
8277
9293
|
for (const [_peer, state] of this._replicationInfoRequestByPeer) {
|
|
8278
9294
|
if (state.timer)
|
|
@@ -8353,7 +9369,9 @@ let SharedLog = (() => {
|
|
|
8353
9369
|
publish: (message, options) => this.rpc.send(new BlocksMessage(message), options),
|
|
8354
9370
|
waitFor: this.rpc.waitFor.bind(this.rpc),
|
|
8355
9371
|
publicKey: this.node.identity.publicKey,
|
|
8356
|
-
|
|
9372
|
+
// Unsolicited block retention is opt-in. Explicit `true` retains the
|
|
9373
|
+
// compatible eager path with bounded validation and storage budgets.
|
|
9374
|
+
eagerBlocks: options?.eagerBlocks ?? false,
|
|
8357
9375
|
resolveProviders: async (cid, opts) => {
|
|
8358
9376
|
// 1) tracker-backed provider directory (best-effort, bounded)
|
|
8359
9377
|
try {
|
|
@@ -9096,10 +10114,13 @@ let SharedLog = (() => {
|
|
|
9096
10114
|
async afterOpen() {
|
|
9097
10115
|
await super.afterOpen();
|
|
9098
10116
|
const existingSubscribersPromise = this._getTopicSubscribers(this.topic);
|
|
10117
|
+
const replicationLifecycleController = this._replicationLifecycleController;
|
|
9099
10118
|
// We do this here, because these calls requires this.closed == false
|
|
9100
10119
|
void this.pruneOfflineReplicators()
|
|
9101
10120
|
.then(() => {
|
|
9102
|
-
this.
|
|
10121
|
+
if (this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
10122
|
+
this._replicatorsReconciled = true;
|
|
10123
|
+
}
|
|
9103
10124
|
})
|
|
9104
10125
|
.catch((error) => {
|
|
9105
10126
|
if (isNotStartedError(error)) {
|
|
@@ -9126,46 +10147,81 @@ let SharedLog = (() => {
|
|
|
9126
10147
|
async pruneOfflineReplicators() {
|
|
9127
10148
|
// Go through all segments and wait for replicators to become reachable;
|
|
9128
10149
|
// otherwise prune them away from the local membership view.
|
|
10150
|
+
const replicationLifecycleController = this._replicationLifecycleController;
|
|
9129
10151
|
try {
|
|
10152
|
+
if (!replicationLifecycleController ||
|
|
10153
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
10154
|
+
return;
|
|
10155
|
+
}
|
|
9130
10156
|
const promises = [];
|
|
9131
10157
|
const iterator = this.replicationIndex.iterate();
|
|
9132
10158
|
const checkedIsAlive = new Set();
|
|
9133
10159
|
while (!iterator.done()) {
|
|
9134
|
-
|
|
10160
|
+
const segments = await iterator.next(1000);
|
|
10161
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
10162
|
+
return;
|
|
10163
|
+
}
|
|
10164
|
+
for (const segment of segments) {
|
|
9135
10165
|
if (checkedIsAlive.has(segment.value.hash) ||
|
|
9136
10166
|
this.node.identity.publicKey.hashcode() === segment.value.hash) {
|
|
9137
10167
|
this.uniqueReplicators.add(this.node.identity.publicKey.hashcode());
|
|
9138
10168
|
continue;
|
|
9139
10169
|
}
|
|
9140
10170
|
checkedIsAlive.add(segment.value.hash);
|
|
9141
|
-
|
|
10171
|
+
const peerHash = segment.value.hash;
|
|
10172
|
+
const subscriptionEpoch = this.getSubscriptionEpoch(peerHash);
|
|
10173
|
+
promises.push(waitForSubscribers(this.node, peerHash, this.rpc.topic, {
|
|
9142
10174
|
timeout: this.waitForReplicatorTimeout,
|
|
9143
10175
|
signal: this._closeController.signal,
|
|
9144
10176
|
})
|
|
9145
10177
|
.then(async () => {
|
|
9146
|
-
|
|
10178
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
10179
|
+
return;
|
|
10180
|
+
}
|
|
10181
|
+
const key = await this._resolvePublicKeyFromHash(peerHash);
|
|
9147
10182
|
if (!key) {
|
|
9148
10183
|
throw new Error("Failed to resolve public key from hash: " +
|
|
9149
|
-
|
|
10184
|
+
peerHash);
|
|
9150
10185
|
}
|
|
9151
10186
|
const keyHash = key.hashcode();
|
|
9152
|
-
|
|
9153
|
-
|
|
9154
|
-
this._replicatorJoinEmitted.add(keyHash);
|
|
9155
|
-
this.events.dispatchEvent(new CustomEvent("replicator:join", {
|
|
9156
|
-
detail: { publicKey: key },
|
|
9157
|
-
}));
|
|
9158
|
-
this.events.dispatchEvent(new CustomEvent("replication:change", {
|
|
9159
|
-
detail: { publicKey: key },
|
|
9160
|
-
}));
|
|
10187
|
+
if (keyHash !== peerHash) {
|
|
10188
|
+
return;
|
|
9161
10189
|
}
|
|
10190
|
+
return this.withReplicationInfoApplyQueue(keyHash, async () => {
|
|
10191
|
+
// A successful reachability check may legitimately span a
|
|
10192
|
+
// subscribe event during startup. The current lane's blocked
|
|
10193
|
+
// state plus an extant index row are the authoritative guard;
|
|
10194
|
+
// only the destructive catch path remains tied to the old token.
|
|
10195
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
10196
|
+
this.closed ||
|
|
10197
|
+
this._replicationInfoBlockedPeers.has(keyHash)) {
|
|
10198
|
+
return;
|
|
10199
|
+
}
|
|
10200
|
+
const hasReplicationRange = (await this.replicationIndex.count({
|
|
10201
|
+
query: { hash: keyHash },
|
|
10202
|
+
})) > 0;
|
|
10203
|
+
if (!hasReplicationRange ||
|
|
10204
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
10205
|
+
this._replicationInfoBlockedPeers.has(keyHash)) {
|
|
10206
|
+
return;
|
|
10207
|
+
}
|
|
10208
|
+
this.uniqueReplicators.add(keyHash);
|
|
10209
|
+
if (!this._replicatorJoinEmitted.has(keyHash)) {
|
|
10210
|
+
this._replicatorJoinEmitted.add(keyHash);
|
|
10211
|
+
this.events.dispatchEvent(new CustomEvent("replicator:join", { detail: { publicKey: key } }));
|
|
10212
|
+
this.events.dispatchEvent(new CustomEvent("replication:change", { detail: { publicKey: key } }));
|
|
10213
|
+
}
|
|
10214
|
+
});
|
|
9162
10215
|
})
|
|
9163
10216
|
.catch(async (error) => {
|
|
9164
|
-
if (isNotStartedError(error)
|
|
10217
|
+
if (isNotStartedError(error) ||
|
|
10218
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
9165
10219
|
return;
|
|
9166
10220
|
}
|
|
9167
|
-
return this.removeReplicator(
|
|
10221
|
+
return this.removeReplicator(peerHash, {
|
|
9168
10222
|
noEvent: true,
|
|
10223
|
+
replicationLifecycleController,
|
|
10224
|
+
subscriptionEpoch,
|
|
9169
10225
|
});
|
|
9170
10226
|
}));
|
|
9171
10227
|
}
|
|
@@ -9173,7 +10229,8 @@ let SharedLog = (() => {
|
|
|
9173
10229
|
return Promise.all(promises);
|
|
9174
10230
|
}
|
|
9175
10231
|
catch (error) {
|
|
9176
|
-
if (isNotStartedError(error)
|
|
10232
|
+
if (isNotStartedError(error) ||
|
|
10233
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
9177
10234
|
return;
|
|
9178
10235
|
}
|
|
9179
10236
|
throw error;
|
|
@@ -9228,10 +10285,26 @@ let SharedLog = (() => {
|
|
|
9228
10285
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
9229
10286
|
this._replicatorLastActivityAt.delete(peerHash);
|
|
9230
10287
|
this._peerSyncCapabilities.delete(peerHash);
|
|
10288
|
+
this.cleanupPendingIHavePeer(peerHash);
|
|
9231
10289
|
this._checkedPrune.cleanupPeer(peerHash);
|
|
9232
10290
|
}
|
|
10291
|
+
cleanupPendingIHavePeer(peerHash) {
|
|
10292
|
+
for (const [hash, pending] of this._pendingIHave) {
|
|
10293
|
+
pending.requesting.delete(peerHash);
|
|
10294
|
+
if (pending.requesting.size === 0) {
|
|
10295
|
+
pending.clear();
|
|
10296
|
+
this._pendingIHave.delete(hash);
|
|
10297
|
+
}
|
|
10298
|
+
}
|
|
10299
|
+
}
|
|
9233
10300
|
markReplicatorActivity(peerHash, now = Date.now()) {
|
|
9234
10301
|
this._replicatorLastActivityAt.set(peerHash, now);
|
|
10302
|
+
// Any recent authenticated activity is positive liveness evidence. Reset the
|
|
10303
|
+
// consecutive miss streak immediately, including while an eviction is
|
|
10304
|
+
// waiting in the per-peer mutation lane.
|
|
10305
|
+
if (Date.now() - now < REPLICATOR_LIVENESS_IDLE_THRESHOLD_MS) {
|
|
10306
|
+
this._replicatorLivenessFailures.delete(peerHash);
|
|
10307
|
+
}
|
|
9235
10308
|
}
|
|
9236
10309
|
hasRecentReplicatorActivity(peerHash, now = Date.now()) {
|
|
9237
10310
|
const lastActivityAt = this._replicatorLastActivityAt.get(peerHash);
|
|
@@ -9242,31 +10315,48 @@ let SharedLog = (() => {
|
|
|
9242
10315
|
}
|
|
9243
10316
|
return false;
|
|
9244
10317
|
}
|
|
9245
|
-
|
|
9246
|
-
|
|
9247
|
-
|
|
9248
|
-
|
|
9249
|
-
|
|
9250
|
-
|
|
9251
|
-
|
|
10318
|
+
advanceReplicationInfoRecoveryEpoch(peerHash) {
|
|
10319
|
+
// Handlers admitted before a successful peer removal must not restore state
|
|
10320
|
+
// when they eventually reach the apply lane. Reset the sender's
|
|
10321
|
+
// ordering watermark with the local epoch so a later arrival can be
|
|
10322
|
+
// accepted without comparing its clock to this receiver's clock.
|
|
10323
|
+
this.advanceReplicationInfoReceiveEpoch(peerHash);
|
|
10324
|
+
this.latestReplicationInfoMessage.delete(peerHash);
|
|
10325
|
+
}
|
|
10326
|
+
async evictReplicatorFromLiveness(peerHash, publicKey, replicationLifecycleController, subscriptionEpoch, observedActivityAt) {
|
|
9252
10327
|
try {
|
|
9253
|
-
await this.removeReplicator(publicKey, {
|
|
10328
|
+
await this.removeReplicator(publicKey, {
|
|
10329
|
+
noEvent: true,
|
|
10330
|
+
replicationLifecycleController,
|
|
10331
|
+
shouldRemove: () => this._replicatorLastActivityAt.get(peerHash) === observedActivityAt,
|
|
10332
|
+
subscriptionEpoch,
|
|
10333
|
+
onRemoved: ({ wasReplicator }) => {
|
|
10334
|
+
if (wasReplicator) {
|
|
10335
|
+
this._pendingReplicatorLeaveByPeer.add(peerHash);
|
|
10336
|
+
}
|
|
10337
|
+
// A newer subscription/lifecycle may have started while the admitted
|
|
10338
|
+
// removal was completing. Its reconnect barrier owns all later effects.
|
|
10339
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
10340
|
+
!this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch)) {
|
|
10341
|
+
return;
|
|
10342
|
+
}
|
|
10343
|
+
if (this._pendingReplicatorLeaveByPeer.delete(peerHash)) {
|
|
10344
|
+
this.events.dispatchEvent(new CustomEvent("replicator:leave", {
|
|
10345
|
+
detail: { publicKey },
|
|
10346
|
+
}));
|
|
10347
|
+
}
|
|
10348
|
+
if (!this._replicationInfoBlockedPeers.has(peerHash)) {
|
|
10349
|
+
this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
|
|
10350
|
+
}
|
|
10351
|
+
this._replicatorLivenessTargetsSize = -1;
|
|
10352
|
+
},
|
|
10353
|
+
});
|
|
9254
10354
|
}
|
|
9255
10355
|
catch (error) {
|
|
9256
10356
|
if (!isNotStartedError(error)) {
|
|
9257
10357
|
throw error;
|
|
9258
10358
|
}
|
|
9259
10359
|
}
|
|
9260
|
-
this.cleanupPeerDisconnectTracking(peerHash);
|
|
9261
|
-
if (wasReplicator) {
|
|
9262
|
-
this.events.dispatchEvent(new CustomEvent("replicator:leave", {
|
|
9263
|
-
detail: { publicKey },
|
|
9264
|
-
}));
|
|
9265
|
-
}
|
|
9266
|
-
if (!this._replicationInfoBlockedPeers.has(peerHash)) {
|
|
9267
|
-
this.scheduleReplicationInfoRequests(publicKey);
|
|
9268
|
-
}
|
|
9269
|
-
this._replicatorLivenessTargetsSize = -1;
|
|
9270
10360
|
}
|
|
9271
10361
|
async resolveCandidatePeersForHash(hash, options) {
|
|
9272
10362
|
if (options?.signal?.aborted)
|
|
@@ -9338,7 +10428,10 @@ let SharedLog = (() => {
|
|
|
9338
10428
|
return pickDeterministicSubset(peers, seed, maxPeers);
|
|
9339
10429
|
}
|
|
9340
10430
|
async runReplicatorLivenessSweep() {
|
|
9341
|
-
|
|
10431
|
+
const replicationLifecycleController = this._replicationLifecycleController;
|
|
10432
|
+
if (this.closed ||
|
|
10433
|
+
this._closeController.signal.aborted ||
|
|
10434
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
9342
10435
|
return;
|
|
9343
10436
|
}
|
|
9344
10437
|
if (this._replicatorLivenessSweepRunning) {
|
|
@@ -9364,13 +10457,23 @@ let SharedLog = (() => {
|
|
|
9364
10457
|
}
|
|
9365
10458
|
}
|
|
9366
10459
|
finally {
|
|
9367
|
-
this.
|
|
10460
|
+
if (this._replicationLifecycleController ===
|
|
10461
|
+
replicationLifecycleController) {
|
|
10462
|
+
this._replicatorLivenessSweepRunning = false;
|
|
10463
|
+
}
|
|
9368
10464
|
}
|
|
9369
10465
|
}
|
|
9370
10466
|
async probeReplicatorLiveness(peerHash) {
|
|
9371
|
-
|
|
10467
|
+
const replicationLifecycleController = this._replicationLifecycleController;
|
|
10468
|
+
if (this.closed ||
|
|
10469
|
+
this._closeController.signal.aborted ||
|
|
10470
|
+
!replicationLifecycleController ||
|
|
10471
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
9372
10472
|
return;
|
|
9373
10473
|
}
|
|
10474
|
+
const subscriptionEpoch = this.getSubscriptionEpoch(peerHash);
|
|
10475
|
+
const ownsProbe = () => this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
10476
|
+
this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch);
|
|
9374
10477
|
if (!this.uniqueReplicators.has(peerHash)) {
|
|
9375
10478
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
9376
10479
|
return;
|
|
@@ -9378,18 +10481,35 @@ let SharedLog = (() => {
|
|
|
9378
10481
|
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
9379
10482
|
return;
|
|
9380
10483
|
}
|
|
10484
|
+
const observedActivityAt = this._replicatorLastActivityAt.get(peerHash);
|
|
9381
10485
|
const publicKey = await this._resolvePublicKeyFromHash(peerHash);
|
|
10486
|
+
if (!ownsProbe()) {
|
|
10487
|
+
return;
|
|
10488
|
+
}
|
|
10489
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
10490
|
+
return;
|
|
10491
|
+
}
|
|
9382
10492
|
if (!publicKey) {
|
|
9383
10493
|
try {
|
|
9384
|
-
await this.removeReplicator(peerHash, {
|
|
10494
|
+
await this.removeReplicator(peerHash, {
|
|
10495
|
+
noEvent: true,
|
|
10496
|
+
replicationLifecycleController,
|
|
10497
|
+
shouldRemove: () => this._replicatorLastActivityAt.get(peerHash) ===
|
|
10498
|
+
observedActivityAt,
|
|
10499
|
+
subscriptionEpoch,
|
|
10500
|
+
onRemoved: () => {
|
|
10501
|
+
if (!ownsProbe()) {
|
|
10502
|
+
return;
|
|
10503
|
+
}
|
|
10504
|
+
this._replicatorLivenessTargetsSize = -1;
|
|
10505
|
+
},
|
|
10506
|
+
});
|
|
9385
10507
|
}
|
|
9386
10508
|
catch (error) {
|
|
9387
10509
|
if (!isNotStartedError(error)) {
|
|
9388
10510
|
throw error;
|
|
9389
10511
|
}
|
|
9390
10512
|
}
|
|
9391
|
-
this.cleanupPeerDisconnectTracking(peerHash);
|
|
9392
|
-
this._replicatorLivenessTargetsSize = -1;
|
|
9393
10513
|
return;
|
|
9394
10514
|
}
|
|
9395
10515
|
try {
|
|
@@ -9400,6 +10520,9 @@ let SharedLog = (() => {
|
|
|
9400
10520
|
priority: ACK_CONTROL_PRIORITY,
|
|
9401
10521
|
responsePriority: ACK_CONTROL_PRIORITY,
|
|
9402
10522
|
});
|
|
10523
|
+
if (!ownsProbe()) {
|
|
10524
|
+
return;
|
|
10525
|
+
}
|
|
9403
10526
|
this.markReplicatorActivity(peerHash);
|
|
9404
10527
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
9405
10528
|
return;
|
|
@@ -9409,25 +10532,40 @@ let SharedLog = (() => {
|
|
|
9409
10532
|
return;
|
|
9410
10533
|
}
|
|
9411
10534
|
}
|
|
10535
|
+
if (!ownsProbe()) {
|
|
10536
|
+
return;
|
|
10537
|
+
}
|
|
10538
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
10539
|
+
return;
|
|
10540
|
+
}
|
|
9412
10541
|
// Relay-backed prod paths can keep a peer subscribed/reachable even if an
|
|
9413
10542
|
// ACKed liveness ping gets delayed or dropped under load. Treat observed
|
|
9414
10543
|
// topic presence as a positive liveness signal before evicting the peer.
|
|
9415
10544
|
if (await this.confirmReplicatorSubscriberPresence(peerHash)) {
|
|
10545
|
+
if (!ownsProbe()) {
|
|
10546
|
+
return;
|
|
10547
|
+
}
|
|
9416
10548
|
this.markReplicatorActivity(peerHash);
|
|
9417
10549
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
9418
10550
|
return;
|
|
9419
10551
|
}
|
|
10552
|
+
if (!ownsProbe()) {
|
|
10553
|
+
return;
|
|
10554
|
+
}
|
|
10555
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
10556
|
+
return;
|
|
10557
|
+
}
|
|
9420
10558
|
const failures = (this._replicatorLivenessFailures.get(peerHash) ?? 0) + 1;
|
|
9421
10559
|
this._replicatorLivenessFailures.set(peerHash, failures);
|
|
9422
|
-
this.scheduleReplicationInfoRequests(publicKey);
|
|
10560
|
+
this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
|
|
9423
10561
|
if (failures < REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT) {
|
|
9424
10562
|
return;
|
|
9425
10563
|
}
|
|
9426
|
-
if (!this.uniqueReplicators.has(peerHash)) {
|
|
10564
|
+
if (!ownsProbe() || !this.uniqueReplicators.has(peerHash)) {
|
|
9427
10565
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
9428
10566
|
return;
|
|
9429
10567
|
}
|
|
9430
|
-
await this.evictReplicatorFromLiveness(peerHash, publicKey);
|
|
10568
|
+
await this.evictReplicatorFromLiveness(peerHash, publicKey, replicationLifecycleController, subscriptionEpoch, observedActivityAt);
|
|
9431
10569
|
}
|
|
9432
10570
|
async confirmReplicatorSubscriberPresence(peerHash) {
|
|
9433
10571
|
try {
|
|
@@ -9459,6 +10597,24 @@ let SharedLog = (() => {
|
|
|
9459
10597
|
return this.log.blocks.size();
|
|
9460
10598
|
/* ((await this.log.entryIndex?.getMemoryUsage()) || 0) */ // + (await this.log.blocks.size())
|
|
9461
10599
|
}
|
|
10600
|
+
/** Return a detached snapshot of effective shared-log runtime settings. */
|
|
10601
|
+
getRuntimeSnapshot() {
|
|
10602
|
+
const nativeGraph = this.log.entryIndex.properties.nativeGraph;
|
|
10603
|
+
const active = nativeGraph?.graph != null;
|
|
10604
|
+
return Object.freeze({
|
|
10605
|
+
nativeGraph: Object.freeze({
|
|
10606
|
+
active,
|
|
10607
|
+
useHeads: active && nativeGraph?.useHeads === true,
|
|
10608
|
+
}),
|
|
10609
|
+
});
|
|
10610
|
+
}
|
|
10611
|
+
/**
|
|
10612
|
+
* Return a detached snapshot of the optional eager-response cache.
|
|
10613
|
+
* Undefined means eager response retention is disabled for this log.
|
|
10614
|
+
*/
|
|
10615
|
+
getEagerBlockCacheTelemetry() {
|
|
10616
|
+
return this.remoteBlocks?.getEagerBlockCacheTelemetry();
|
|
10617
|
+
}
|
|
9462
10618
|
clampReplicas(value) {
|
|
9463
10619
|
const lower = this.replicas.min?.getValue(this) || 1;
|
|
9464
10620
|
const higher = this.replicas.max?.getValue(this) ?? Number.MAX_SAFE_INTEGER;
|
|
@@ -10073,6 +11229,7 @@ let SharedLog = (() => {
|
|
|
10073
11229
|
captureSync(() => this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn));
|
|
10074
11230
|
captureSync(() => this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn));
|
|
10075
11231
|
captureSync(() => {
|
|
11232
|
+
this.cancelAllJoinWarmupTargets();
|
|
10076
11233
|
for (const timer of this._repairRetryTimers ?? [])
|
|
10077
11234
|
clearTimeout(timer);
|
|
10078
11235
|
this._repairRetryTimers?.clear();
|
|
@@ -10081,7 +11238,9 @@ let SharedLog = (() => {
|
|
|
10081
11238
|
this._repairSweepPendingModes?.clear();
|
|
10082
11239
|
for (const peers of this._repairSweepPendingPeersByMode?.values() ?? [])
|
|
10083
11240
|
peers.clear();
|
|
11241
|
+
this._repairSweepJoinWarmupGenerationByTarget?.clear();
|
|
10084
11242
|
this._repairSweepOptimisticGidPeersPending?.clear();
|
|
11243
|
+
this._repairSweepOptimisticGidsByPeer?.clear();
|
|
10085
11244
|
this._entryKnownPeers?.clear();
|
|
10086
11245
|
this._entryKnownPeerObservedAt?.clear();
|
|
10087
11246
|
this._nativeSharedLogState?.clearEntryKnownPeers();
|
|
@@ -10118,7 +11277,14 @@ let SharedLog = (() => {
|
|
|
10118
11277
|
}
|
|
10119
11278
|
captureSync(() => {
|
|
10120
11279
|
this._pendingIHave?.clear();
|
|
11280
|
+
this._pendingIHaveCallbacks?.clear();
|
|
10121
11281
|
this.latestReplicationInfoMessage?.clear();
|
|
11282
|
+
this._replicationInfoReceiveEpochByPeer?.clear();
|
|
11283
|
+
this._activeReceiveHandlersByPeer?.clear();
|
|
11284
|
+
this._receiveHandlerDrainByPeer?.clear();
|
|
11285
|
+
this._receiveCleanupGateByPeer?.clear();
|
|
11286
|
+
this._subscriptionOpeningEpochByPeer?.clear();
|
|
11287
|
+
this._openingSyncCapabilitiesByPeer?.clear();
|
|
10122
11288
|
this._gidPeersHistory?.clear();
|
|
10123
11289
|
this._peerSyncCapabilities?.clear();
|
|
10124
11290
|
this._liveRawGossipBatches?.clear();
|
|
@@ -10181,7 +11347,14 @@ let SharedLog = (() => {
|
|
|
10181
11347
|
// SharedLog-specific await or observable teardown can admit a new owner.
|
|
10182
11348
|
this.preventParentAttachments();
|
|
10183
11349
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
11350
|
+
this.cancelAllJoinWarmupTargets();
|
|
10184
11351
|
await this.drainSubscriptionChangeCallbacks();
|
|
11352
|
+
// An already-admitted subscription callback can create a fresh warmup
|
|
11353
|
+
// generation while the first cancellation is draining.
|
|
11354
|
+
this.cancelAllJoinWarmupTargets();
|
|
11355
|
+
await this.drainReceiveHandlers();
|
|
11356
|
+
await this.drainReplicationInfoApplyQueues();
|
|
11357
|
+
await this.drainPendingIHaveCallbacks();
|
|
10185
11358
|
this.ensureNativeDurabilityRuntimeState();
|
|
10186
11359
|
this.cancelCurrentReplicationStateAnnouncementRetry();
|
|
10187
11360
|
// Best-effort: announce that we are going offline before tearing down
|
|
@@ -10301,7 +11474,14 @@ let SharedLog = (() => {
|
|
|
10301
11474
|
// the terminal fence only after that precondition succeeds.
|
|
10302
11475
|
this.preventParentAttachments();
|
|
10303
11476
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
11477
|
+
this.cancelAllJoinWarmupTargets();
|
|
10304
11478
|
await this.drainSubscriptionChangeCallbacks();
|
|
11479
|
+
// An already-admitted subscription callback can create a fresh warmup
|
|
11480
|
+
// generation while the first cancellation is draining.
|
|
11481
|
+
this.cancelAllJoinWarmupTargets();
|
|
11482
|
+
await this.drainReceiveHandlers();
|
|
11483
|
+
await this.drainReplicationInfoApplyQueues();
|
|
11484
|
+
await this.drainPendingIHaveCallbacks();
|
|
10305
11485
|
this.cancelCurrentReplicationStateAnnouncementRetry();
|
|
10306
11486
|
// Best-effort: announce that we are going offline before tearing down
|
|
10307
11487
|
// RPC/subscription state (same reasoning as in `close()`).
|
|
@@ -10512,13 +11692,35 @@ let SharedLog = (() => {
|
|
|
10512
11692
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
10513
11693
|
? msg
|
|
10514
11694
|
: undefined;
|
|
11695
|
+
let releasePeerReceiveLease;
|
|
10515
11696
|
try {
|
|
10516
11697
|
this.throwIfNativeDurableCommitFailed();
|
|
10517
11698
|
if (!context.from) {
|
|
10518
11699
|
throw new Error("Missing from in update role message");
|
|
10519
11700
|
}
|
|
11701
|
+
// Snapshot receive ownership before any async handler gets a chance to
|
|
11702
|
+
// yield. Replication-info messages reach their branch only after the
|
|
11703
|
+
// synchronizer declines them, and a U/S transition can happen meanwhile.
|
|
11704
|
+
const receiveFromHash = context.from.hashcode();
|
|
11705
|
+
const receiveReplicationLifecycleController = this._replicationLifecycleController;
|
|
11706
|
+
const receiveSubscriptionEpoch = this.getSubscriptionEpoch(receiveFromHash);
|
|
11707
|
+
const isOpeningSubscriptionReceive = this._subscriptionOpeningEpochByPeer.get(receiveFromHash) ===
|
|
11708
|
+
receiveSubscriptionEpoch;
|
|
11709
|
+
const isOpeningCapabilityAdvertisement = msg instanceof SyncCapabilitiesMessage && isOpeningSubscriptionReceive;
|
|
11710
|
+
releasePeerReceiveLease = this.acquirePeerReceiveLease(receiveFromHash, receiveReplicationLifecycleController, receiveSubscriptionEpoch, {
|
|
11711
|
+
// The replication-info fence existed before receive leases and is
|
|
11712
|
+
// intentionally narrower than the subscription itself. Keep admitting
|
|
11713
|
+
// sync negotiation/data while the new subscription drains the previous
|
|
11714
|
+
// apply generation; replication-info branches recheck the fence below.
|
|
11715
|
+
allowReplicationInfoBlocked: isOpeningSubscriptionReceive,
|
|
11716
|
+
allowCleanupGate: isOpeningCapabilityAdvertisement,
|
|
11717
|
+
});
|
|
11718
|
+
if (!releasePeerReceiveLease) {
|
|
11719
|
+
return;
|
|
11720
|
+
}
|
|
11721
|
+
const receiveReplicationInfoReceiveEpoch = this.getReplicationInfoReceiveEpoch(receiveFromHash);
|
|
10520
11722
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
10521
|
-
this.markReplicatorActivity(
|
|
11723
|
+
this.markReplicatorActivity(receiveFromHash);
|
|
10522
11724
|
}
|
|
10523
11725
|
if (msg instanceof ResponseRoleMessage) {
|
|
10524
11726
|
msg = msg.toReplicationInfoMessage(); // migration
|
|
@@ -11601,6 +12803,7 @@ let SharedLog = (() => {
|
|
|
11601
12803
|
allToMergeMaterializedEntries ??= allToMerge.map((entry) => entry.entry);
|
|
11602
12804
|
return allToMergeMaterializedEntries;
|
|
11603
12805
|
};
|
|
12806
|
+
let admittedMergeHashes = new Set();
|
|
11604
12807
|
let nativePreparedCommittedHashes;
|
|
11605
12808
|
if (allToMerge.length > 0) {
|
|
11606
12809
|
const validateStartedAt = syncProfileStart(syncProfile);
|
|
@@ -11696,8 +12899,8 @@ let SharedLog = (() => {
|
|
|
11696
12899
|
const entriesToPersist = usedNativeAllKeptJoinPlan
|
|
11697
12900
|
? allToMergeShallowEntries
|
|
11698
12901
|
: joinPlans.flatMap((plan) => plan.toPersist);
|
|
11699
|
-
|
|
11700
|
-
|
|
12902
|
+
let coordinatePersistFallbackEntries = [];
|
|
12903
|
+
let reusableCoordinatePersistItems = [];
|
|
11701
12904
|
for (const entry of entriesToPersist) {
|
|
11702
12905
|
const reusablePlan = reusableCoordinatePlans.get(entry.hash);
|
|
11703
12906
|
if (!reusablePlan) {
|
|
@@ -11713,7 +12916,6 @@ let SharedLog = (() => {
|
|
|
11713
12916
|
prepared: reusablePlan.prepared,
|
|
11714
12917
|
});
|
|
11715
12918
|
}
|
|
11716
|
-
const reusableCoordinatePersistItemCount = reusableCoordinatePersistItems.length;
|
|
11717
12919
|
let nativePreparedCoordinateBatch;
|
|
11718
12920
|
let nativePreparedCoordinatesFinished = false;
|
|
11719
12921
|
let nativeBackboneOnlyPersistedHashes;
|
|
@@ -11789,6 +12991,26 @@ let SharedLog = (() => {
|
|
|
11789
12991
|
__peerbitProfile: syncProfile,
|
|
11790
12992
|
});
|
|
11791
12993
|
}
|
|
12994
|
+
// A recursive lower-log join can resolve successfully while declining
|
|
12995
|
+
// an individual top-level entry (for example, when one of its parents
|
|
12996
|
+
// is temporarily unavailable). The public Log.join() API intentionally
|
|
12997
|
+
// does not expose that per-entry result, so make local index presence the
|
|
12998
|
+
// authority before publishing any SharedLog-side effects. A successful
|
|
12999
|
+
// prepared-facts batch is atomic and already proves every input hash.
|
|
13000
|
+
const admittedHashes = joinedPreparedFacts
|
|
13001
|
+
? new Set(allToMergeHashes)
|
|
13002
|
+
: await this.log.hasMany(allToMergeHashes);
|
|
13003
|
+
admittedMergeHashes = admittedHashes;
|
|
13004
|
+
const admittedShallowEntries = admittedHashes.size === allToMergeShallowEntries.length
|
|
13005
|
+
? allToMergeShallowEntries
|
|
13006
|
+
: allToMergeShallowEntries.filter((entry) => admittedHashes.has(entry.hash));
|
|
13007
|
+
if (!joinedPreparedFacts) {
|
|
13008
|
+
reusableCoordinatePersistItems =
|
|
13009
|
+
reusableCoordinatePersistItems.filter((item) => admittedHashes.has(item.entry.hash));
|
|
13010
|
+
coordinatePersistFallbackEntries =
|
|
13011
|
+
coordinatePersistFallbackEntries.filter((entry) => admittedHashes.has(entry.hash));
|
|
13012
|
+
}
|
|
13013
|
+
const reusableCoordinatePersistItemCount = reusableCoordinatePersistItems.length;
|
|
11792
13014
|
if (syncProfile) {
|
|
11793
13015
|
emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
|
|
11794
13016
|
name: "sharedLog.receive.lowerLogJoin",
|
|
@@ -11800,6 +13022,7 @@ let SharedLog = (() => {
|
|
|
11800
13022
|
batchHashOnlyEntryAdded,
|
|
11801
13023
|
programOnChange,
|
|
11802
13024
|
joinedPreparedFacts,
|
|
13025
|
+
admittedEntries: admittedHashes.size,
|
|
11803
13026
|
nativePreparedCoordinatesFinished,
|
|
11804
13027
|
},
|
|
11805
13028
|
});
|
|
@@ -11854,11 +13077,11 @@ let SharedLog = (() => {
|
|
|
11854
13077
|
},
|
|
11855
13078
|
});
|
|
11856
13079
|
}
|
|
11857
|
-
for (const hash of
|
|
13080
|
+
for (const hash of admittedHashes) {
|
|
11858
13081
|
confirmedHashes.add(hash);
|
|
11859
13082
|
}
|
|
11860
13083
|
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
11861
|
-
await this.pruneJoinedEntriesNoLongerLed(
|
|
13084
|
+
await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
|
|
11862
13085
|
decodedReplicaCounts: receiveReplicaCounts,
|
|
11863
13086
|
reusableLeaderPlans: reusableCoordinatePlans,
|
|
11864
13087
|
profile: syncProfile,
|
|
@@ -11872,10 +13095,12 @@ let SharedLog = (() => {
|
|
|
11872
13095
|
});
|
|
11873
13096
|
}
|
|
11874
13097
|
for (const plan of joinPlans) {
|
|
11875
|
-
plan.toDelete
|
|
11876
|
-
|
|
13098
|
+
plan.toDelete
|
|
13099
|
+
?.filter((entry) => admittedMergeHashes.has(entry.hash))
|
|
13100
|
+
.map((entry) => this.pruneDebouncedFnAddIfNotKeeping({
|
|
13101
|
+
key: entry.hash,
|
|
11877
13102
|
value: {
|
|
11878
|
-
entry
|
|
13103
|
+
entry,
|
|
11879
13104
|
leaders: plan.leaders,
|
|
11880
13105
|
},
|
|
11881
13106
|
}));
|
|
@@ -11887,14 +13112,18 @@ let SharedLog = (() => {
|
|
|
11887
13112
|
continue;
|
|
11888
13113
|
}
|
|
11889
13114
|
for (const entries of plan.maybeDelete) {
|
|
11890
|
-
const
|
|
13115
|
+
const admittedEntries = entries.filter((entry) => admittedMergeHashes.has(getExchangeHeadHash(entry)));
|
|
13116
|
+
if (admittedEntries.length === 0) {
|
|
13117
|
+
continue;
|
|
13118
|
+
}
|
|
13119
|
+
const minReplicas = await this.getMaxReplicasFromHeads(this.getEntryGid(admittedEntries[0].entry));
|
|
11891
13120
|
if (minReplicas != null) {
|
|
11892
13121
|
const isLeader = await this.isLeader({
|
|
11893
|
-
entry:
|
|
13122
|
+
entry: admittedEntries[0].entry,
|
|
11894
13123
|
replicas: minReplicas,
|
|
11895
13124
|
});
|
|
11896
13125
|
if (!isLeader) {
|
|
11897
|
-
for (const x of
|
|
13126
|
+
for (const x of admittedEntries) {
|
|
11898
13127
|
this.pruneDebouncedFnAddIfNotKeeping({
|
|
11899
13128
|
key: x.entry.hash,
|
|
11900
13129
|
value: {
|
|
@@ -12101,6 +13330,9 @@ let SharedLog = (() => {
|
|
|
12101
13330
|
let pendingIHaveExtended = 0;
|
|
12102
13331
|
const leaderResponseHashes = [];
|
|
12103
13332
|
for (let i = 0; i < msg.hashes.length; i++) {
|
|
13333
|
+
if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController)) {
|
|
13334
|
+
return;
|
|
13335
|
+
}
|
|
12104
13336
|
const hash = msg.hashes[i];
|
|
12105
13337
|
const nativeEntryGid = nativeLeaderHints.nativeEntryGids?.[i];
|
|
12106
13338
|
const nativeEntryData = nativeLeaderHints.nativeEntryDataByIndex?.[i];
|
|
@@ -12213,8 +13445,14 @@ let SharedLog = (() => {
|
|
|
12213
13445
|
resetTimeout: () => this.resetPendingIHaveTimeout(pendingIHave),
|
|
12214
13446
|
clear: () => this.clearPendingIHaveTimeout(pendingIHave),
|
|
12215
13447
|
callback: async (entry) => {
|
|
12216
|
-
this.
|
|
12217
|
-
|
|
13448
|
+
if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
|
|
13449
|
+
requesting.size === 0) {
|
|
13450
|
+
return;
|
|
13451
|
+
}
|
|
13452
|
+
for (const requester of requesting) {
|
|
13453
|
+
this.removePeerFromGidPeerHistory(requester, entry.meta.gid);
|
|
13454
|
+
this.removePruneRequestSent(entry.hash, requester);
|
|
13455
|
+
}
|
|
12218
13456
|
let isLeader = false;
|
|
12219
13457
|
await this._waitForEntryReplicators(entry, decodeReplicas(entry).getValue(this), [
|
|
12220
13458
|
{
|
|
@@ -12228,12 +13466,15 @@ let SharedLog = (() => {
|
|
|
12228
13466
|
key === this.node.identity.publicKey.hashcode();
|
|
12229
13467
|
},
|
|
12230
13468
|
});
|
|
13469
|
+
if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
|
|
13470
|
+
requesting.size === 0) {
|
|
13471
|
+
return;
|
|
13472
|
+
}
|
|
12231
13473
|
if (isLeader) {
|
|
12232
13474
|
this.responseToPruneDebouncedFn.add({
|
|
12233
13475
|
hashes: [entry.hash],
|
|
12234
|
-
peers: requesting,
|
|
13476
|
+
peers: new Set(requesting),
|
|
12235
13477
|
});
|
|
12236
|
-
this._pendingIHave.delete(hash);
|
|
12237
13478
|
}
|
|
12238
13479
|
},
|
|
12239
13480
|
};
|
|
@@ -12276,17 +13517,24 @@ let SharedLog = (() => {
|
|
|
12276
13517
|
}
|
|
12277
13518
|
else if (msg instanceof ResponseIPrune) {
|
|
12278
13519
|
const lateResponses = [];
|
|
13520
|
+
const responseTasks = [];
|
|
12279
13521
|
for (const hash of msg.hashes) {
|
|
12280
13522
|
const pendingDelete = this._checkedPrune.getPendingDelete(hash);
|
|
12281
13523
|
if (pendingDelete) {
|
|
12282
|
-
|
|
13524
|
+
responseTasks.push(Promise.resolve(pendingDelete.resolve(context.from.hashcode())));
|
|
12283
13525
|
}
|
|
12284
13526
|
else {
|
|
12285
13527
|
lateResponses.push(hash);
|
|
12286
13528
|
}
|
|
12287
13529
|
}
|
|
12288
13530
|
if (lateResponses.length > 0) {
|
|
12289
|
-
|
|
13531
|
+
responseTasks.push(this.recoverCheckedPruneFromLateResponses(lateResponses, context.from.hashcode()));
|
|
13532
|
+
}
|
|
13533
|
+
const results = await Promise.allSettled(responseTasks);
|
|
13534
|
+
for (const result of results) {
|
|
13535
|
+
if (result.status === "rejected") {
|
|
13536
|
+
logger.error(result.reason?.toString?.() ?? String(result.reason));
|
|
13537
|
+
}
|
|
12290
13538
|
}
|
|
12291
13539
|
}
|
|
12292
13540
|
else if (msg instanceof ConfirmEntriesMessage) {
|
|
@@ -12296,7 +13544,20 @@ let SharedLog = (() => {
|
|
|
12296
13544
|
}
|
|
12297
13545
|
else if (msg instanceof SyncCapabilitiesMessage) {
|
|
12298
13546
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
12299
|
-
this.
|
|
13547
|
+
const openingEpoch = this._subscriptionOpeningEpochByPeer.get(receiveFromHash);
|
|
13548
|
+
if (this._replicationInfoBlockedPeers.has(receiveFromHash) &&
|
|
13549
|
+
openingEpoch === receiveSubscriptionEpoch) {
|
|
13550
|
+
// A prior unsubscribe cleanup may still be ahead of this reconnect
|
|
13551
|
+
// barrier. Stage the new generation's one-shot advertisement so that
|
|
13552
|
+
// cleanup cannot erase it before the opening transition commits.
|
|
13553
|
+
this._openingSyncCapabilitiesByPeer.set(receiveFromHash, {
|
|
13554
|
+
epoch: openingEpoch,
|
|
13555
|
+
capabilities: msg.capabilities,
|
|
13556
|
+
});
|
|
13557
|
+
}
|
|
13558
|
+
else {
|
|
13559
|
+
this._peerSyncCapabilities.set(receiveFromHash, msg.capabilities);
|
|
13560
|
+
}
|
|
12300
13561
|
}
|
|
12301
13562
|
return;
|
|
12302
13563
|
}
|
|
@@ -12316,9 +13577,9 @@ let SharedLog = (() => {
|
|
|
12316
13577
|
if (context.from.equals(this.node.identity.publicKey)) {
|
|
12317
13578
|
return;
|
|
12318
13579
|
}
|
|
12319
|
-
const replicationLifecycleController =
|
|
13580
|
+
const replicationLifecycleController = receiveReplicationLifecycleController;
|
|
12320
13581
|
if (!replicationLifecycleController ||
|
|
12321
|
-
!this.
|
|
13582
|
+
!this.isPeerReceiveAdmissionOpen(receiveFromHash, replicationLifecycleController, receiveSubscriptionEpoch)) {
|
|
12322
13583
|
return;
|
|
12323
13584
|
}
|
|
12324
13585
|
let replicationSegments;
|
|
@@ -12326,13 +13587,13 @@ let SharedLog = (() => {
|
|
|
12326
13587
|
replicationSegments = await this.getMyReplicationSegments();
|
|
12327
13588
|
}
|
|
12328
13589
|
catch (error) {
|
|
12329
|
-
if (!this.
|
|
13590
|
+
if (!this.isPeerReceiveAdmissionOpen(receiveFromHash, replicationLifecycleController, receiveSubscriptionEpoch) &&
|
|
12330
13591
|
isNotStartedError(error)) {
|
|
12331
13592
|
return;
|
|
12332
13593
|
}
|
|
12333
13594
|
throw error;
|
|
12334
13595
|
}
|
|
12335
|
-
if (!this.
|
|
13596
|
+
if (!this.isPeerReceiveAdmissionOpen(receiveFromHash, replicationLifecycleController, receiveSubscriptionEpoch)) {
|
|
12336
13597
|
return;
|
|
12337
13598
|
}
|
|
12338
13599
|
const segments = replicationSegments.map((x) => x.toReplicationRange());
|
|
@@ -12345,7 +13606,7 @@ let SharedLog = (() => {
|
|
|
12345
13606
|
signal: replicationLifecycleController.signal,
|
|
12346
13607
|
})
|
|
12347
13608
|
.catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
|
|
12348
|
-
if (!this.
|
|
13609
|
+
if (!this.isPeerReceiveAdmissionOpen(receiveFromHash, replicationLifecycleController, receiveSubscriptionEpoch)) {
|
|
12349
13610
|
return;
|
|
12350
13611
|
}
|
|
12351
13612
|
// for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
|
|
@@ -12380,14 +13641,21 @@ let SharedLog = (() => {
|
|
|
12380
13641
|
// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
|
|
12381
13642
|
const from = context.from;
|
|
12382
13643
|
const fromHash = from.hashcode();
|
|
12383
|
-
if (this.
|
|
13644
|
+
if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
|
|
13645
|
+
!this.isCurrentReplicationInfoReceiveEpoch(receiveFromHash, receiveReplicationInfoReceiveEpoch) ||
|
|
13646
|
+
this._replicationInfoBlockedPeers.has(fromHash)) {
|
|
12384
13647
|
return;
|
|
12385
13648
|
}
|
|
12386
13649
|
const messageTimestamp = context.message.header.timestamp;
|
|
13650
|
+
releasePeerReceiveLease?.();
|
|
13651
|
+
releasePeerReceiveLease = undefined;
|
|
12387
13652
|
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
12388
13653
|
try {
|
|
12389
13654
|
// The peer may have unsubscribed after this message was queued.
|
|
12390
|
-
if (this.
|
|
13655
|
+
if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
|
|
13656
|
+
!this.isCurrentSubscriptionEpoch(fromHash, receiveSubscriptionEpoch) ||
|
|
13657
|
+
!this.isCurrentReplicationInfoReceiveEpoch(fromHash, receiveReplicationInfoReceiveEpoch) ||
|
|
13658
|
+
this._replicationInfoBlockedPeers.has(fromHash)) {
|
|
12391
13659
|
return;
|
|
12392
13660
|
}
|
|
12393
13661
|
// Process in-order to avoid races where repeated reset messages arrive
|
|
@@ -12422,24 +13690,47 @@ let SharedLog = (() => {
|
|
|
12422
13690
|
});
|
|
12423
13691
|
}
|
|
12424
13692
|
else if (msg instanceof StoppedReplicating) {
|
|
12425
|
-
|
|
13693
|
+
const from = context.from;
|
|
13694
|
+
const segmentIds = msg.segmentIds;
|
|
13695
|
+
if (from.equals(this.node.identity.publicKey)) {
|
|
12426
13696
|
return;
|
|
12427
13697
|
}
|
|
12428
|
-
const fromHash =
|
|
12429
|
-
if (this.
|
|
13698
|
+
const fromHash = from.hashcode();
|
|
13699
|
+
if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
|
|
13700
|
+
!this.isCurrentReplicationInfoReceiveEpoch(receiveFromHash, receiveReplicationInfoReceiveEpoch) ||
|
|
13701
|
+
this._replicationInfoBlockedPeers.has(fromHash)) {
|
|
12430
13702
|
return;
|
|
12431
13703
|
}
|
|
12432
|
-
|
|
12433
|
-
|
|
12434
|
-
|
|
12435
|
-
|
|
12436
|
-
|
|
12437
|
-
|
|
12438
|
-
|
|
12439
|
-
|
|
12440
|
-
|
|
12441
|
-
}
|
|
12442
|
-
|
|
13704
|
+
const messageTimestamp = context.message.header.timestamp;
|
|
13705
|
+
releasePeerReceiveLease?.();
|
|
13706
|
+
releasePeerReceiveLease = undefined;
|
|
13707
|
+
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
13708
|
+
if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
|
|
13709
|
+
!this.isCurrentSubscriptionEpoch(fromHash, receiveSubscriptionEpoch) ||
|
|
13710
|
+
!this.isCurrentReplicationInfoReceiveEpoch(fromHash, receiveReplicationInfoReceiveEpoch) ||
|
|
13711
|
+
this._replicationInfoBlockedPeers.has(fromHash)) {
|
|
13712
|
+
return;
|
|
13713
|
+
}
|
|
13714
|
+
const previousTimestamp = this.latestReplicationInfoMessage.get(fromHash);
|
|
13715
|
+
if (previousTimestamp && previousTimestamp > messageTimestamp) {
|
|
13716
|
+
return;
|
|
13717
|
+
}
|
|
13718
|
+
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
13719
|
+
this._replicatorLivenessFailures.delete(fromHash);
|
|
13720
|
+
if (this.closed) {
|
|
13721
|
+
return;
|
|
13722
|
+
}
|
|
13723
|
+
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(segmentIds, from);
|
|
13724
|
+
await this.removeReplicationRanges(rangesToRemove, from);
|
|
13725
|
+
const timestamp = BigInt(+new Date());
|
|
13726
|
+
for (const range of rangesToRemove) {
|
|
13727
|
+
this.replicationChangeDebounceFn.add({
|
|
13728
|
+
range,
|
|
13729
|
+
type: "removed",
|
|
13730
|
+
timestamp,
|
|
13731
|
+
});
|
|
13732
|
+
}
|
|
13733
|
+
});
|
|
12443
13734
|
}
|
|
12444
13735
|
else {
|
|
12445
13736
|
throw new Error("Unexpected message");
|
|
@@ -12485,6 +13776,8 @@ let SharedLog = (() => {
|
|
|
12485
13776
|
// Every return and every locally swallowed receive error passes this
|
|
12486
13777
|
// boundary. Release a native wire stash exactly once first, then surface
|
|
12487
13778
|
// any durable mutation poison that arose while handling the message.
|
|
13779
|
+
releasePeerReceiveLease?.();
|
|
13780
|
+
releasePeerReceiveLease = undefined;
|
|
12488
13781
|
this.throwIfNativeDurableCommitFailed();
|
|
12489
13782
|
}
|
|
12490
13783
|
}
|
|
@@ -12960,12 +14253,16 @@ let SharedLog = (() => {
|
|
|
12960
14253
|
}
|
|
12961
14254
|
async waitForLeaderSelection(waitFor, options, checkLeaders) {
|
|
12962
14255
|
const timeout = options.timeout ?? this.waitForReplicatorTimeout;
|
|
14256
|
+
const closeSignal = this._closeController.signal;
|
|
14257
|
+
const replicationLifecycleSignal = this._replicationLifecycleController?.signal;
|
|
12963
14258
|
return new Promise((resolve, reject) => {
|
|
12964
14259
|
let settled = false;
|
|
14260
|
+
const checks = new Set();
|
|
12965
14261
|
const removeListeners = () => {
|
|
12966
14262
|
this.events.removeEventListener("replication:change", roleListener);
|
|
12967
14263
|
this.events.removeEventListener("replicator:mature", roleListener);
|
|
12968
|
-
|
|
14264
|
+
closeSignal.removeEventListener("abort", abortListener);
|
|
14265
|
+
replicationLifecycleSignal?.removeEventListener("abort", abortListener);
|
|
12969
14266
|
};
|
|
12970
14267
|
const settleResolve = (value) => {
|
|
12971
14268
|
if (settled)
|
|
@@ -12973,7 +14270,10 @@ let SharedLog = (() => {
|
|
|
12973
14270
|
settled = true;
|
|
12974
14271
|
removeListeners();
|
|
12975
14272
|
clearTimeout(timer);
|
|
12976
|
-
|
|
14273
|
+
// Leader planning may persist coordinates. Keep the caller (and any
|
|
14274
|
+
// receive lease it owns) alive until checks admitted before this
|
|
14275
|
+
// timeout/abort have finished their local side effects.
|
|
14276
|
+
void Promise.allSettled([...checks]).then(() => resolve(value));
|
|
12977
14277
|
};
|
|
12978
14278
|
const settleReject = (error) => {
|
|
12979
14279
|
if (settled)
|
|
@@ -12981,7 +14281,7 @@ let SharedLog = (() => {
|
|
|
12981
14281
|
settled = true;
|
|
12982
14282
|
removeListeners();
|
|
12983
14283
|
clearTimeout(timer);
|
|
12984
|
-
reject(error);
|
|
14284
|
+
void Promise.allSettled([...checks]).then(() => reject(error));
|
|
12985
14285
|
};
|
|
12986
14286
|
const abortListener = () => {
|
|
12987
14287
|
settleResolve(false);
|
|
@@ -13010,16 +14310,30 @@ let SharedLog = (() => {
|
|
|
13010
14310
|
settleResolve(leaders);
|
|
13011
14311
|
};
|
|
13012
14312
|
const runCheck = () => {
|
|
13013
|
-
|
|
14313
|
+
if (settled)
|
|
14314
|
+
return;
|
|
14315
|
+
let running;
|
|
14316
|
+
running = check()
|
|
14317
|
+
.catch((error) => {
|
|
13014
14318
|
settleReject(error);
|
|
13015
|
-
})
|
|
14319
|
+
})
|
|
14320
|
+
.finally(() => checks.delete(running));
|
|
14321
|
+
checks.add(running);
|
|
13016
14322
|
};
|
|
13017
14323
|
const roleListener = () => {
|
|
13018
14324
|
runCheck();
|
|
13019
14325
|
};
|
|
13020
14326
|
this.events.addEventListener("replication:change", roleListener);
|
|
13021
14327
|
this.events.addEventListener("replicator:mature", roleListener);
|
|
13022
|
-
|
|
14328
|
+
closeSignal.addEventListener("abort", abortListener);
|
|
14329
|
+
replicationLifecycleSignal?.addEventListener("abort", abortListener);
|
|
14330
|
+
// AbortSignal does not replay an abort event to listeners added after it
|
|
14331
|
+
// fired. Recheck after registration so work started concurrently with the
|
|
14332
|
+
// terminal fence cannot wait for the full leader-selection timeout.
|
|
14333
|
+
if (closeSignal.aborted || replicationLifecycleSignal?.aborted) {
|
|
14334
|
+
abortListener();
|
|
14335
|
+
return;
|
|
14336
|
+
}
|
|
13023
14337
|
runCheck();
|
|
13024
14338
|
});
|
|
13025
14339
|
}
|
|
@@ -15054,6 +16368,40 @@ let SharedLog = (() => {
|
|
|
15054
16368
|
}
|
|
15055
16369
|
});
|
|
15056
16370
|
}
|
|
16371
|
+
async drainReplicationInfoApplyQueues() {
|
|
16372
|
+
for (;;) {
|
|
16373
|
+
const tails = [...(this._replicationInfoApplyQueueByPeer?.values() ?? [])];
|
|
16374
|
+
if (tails.length === 0) {
|
|
16375
|
+
return;
|
|
16376
|
+
}
|
|
16377
|
+
await Promise.allSettled(tails);
|
|
16378
|
+
// Queue cleanup runs in `finally`; give it a microtask before checking for
|
|
16379
|
+
// tails admitted while the previous snapshot was settling.
|
|
16380
|
+
await Promise.resolve();
|
|
16381
|
+
}
|
|
16382
|
+
}
|
|
16383
|
+
advanceReplicationInfoReceiveEpoch(peerHash) {
|
|
16384
|
+
const next = {};
|
|
16385
|
+
this._replicationInfoReceiveEpochByPeer.set(peerHash, next);
|
|
16386
|
+
return next;
|
|
16387
|
+
}
|
|
16388
|
+
getReplicationInfoReceiveEpoch(peerHash) {
|
|
16389
|
+
return this._replicationInfoReceiveEpochByPeer.get(peerHash) ?? null;
|
|
16390
|
+
}
|
|
16391
|
+
isCurrentReplicationInfoReceiveEpoch(peerHash, epoch) {
|
|
16392
|
+
return this.getReplicationInfoReceiveEpoch(peerHash) === epoch;
|
|
16393
|
+
}
|
|
16394
|
+
advanceSubscriptionEpoch(peerHash) {
|
|
16395
|
+
const next = {};
|
|
16396
|
+
this._subscriptionEpochByPeer.set(peerHash, next);
|
|
16397
|
+
return next;
|
|
16398
|
+
}
|
|
16399
|
+
getSubscriptionEpoch(peerHash) {
|
|
16400
|
+
return this._subscriptionEpochByPeer.get(peerHash) ?? null;
|
|
16401
|
+
}
|
|
16402
|
+
isCurrentSubscriptionEpoch(peerHash, epoch) {
|
|
16403
|
+
return this.getSubscriptionEpoch(peerHash) === epoch;
|
|
16404
|
+
}
|
|
15057
16405
|
cancelReplicationInfoRequests(peerHash) {
|
|
15058
16406
|
const state = this._replicationInfoRequestByPeer.get(peerHash);
|
|
15059
16407
|
if (!state)
|
|
@@ -15118,7 +16466,7 @@ let SharedLog = (() => {
|
|
|
15118
16466
|
};
|
|
15119
16467
|
tick();
|
|
15120
16468
|
}
|
|
15121
|
-
async handleSubscriptionChange(publicKey, topics, subscribed) {
|
|
16469
|
+
async handleSubscriptionChange(publicKey, topics, subscribed, subscriptionEpoch) {
|
|
15122
16470
|
if (!topics.includes(this.topic)) {
|
|
15123
16471
|
return;
|
|
15124
16472
|
}
|
|
@@ -15128,30 +16476,91 @@ let SharedLog = (() => {
|
|
|
15128
16476
|
return;
|
|
15129
16477
|
}
|
|
15130
16478
|
const peerHash = publicKey.hashcode();
|
|
16479
|
+
const expectedSubscriptionEpoch = subscriptionEpoch ?? this.advanceSubscriptionEpoch(peerHash);
|
|
16480
|
+
const ownsSubscriptionEpoch = () => this.isCurrentSubscriptionEpoch(peerHash, expectedSubscriptionEpoch);
|
|
16481
|
+
if (!ownsSubscriptionEpoch()) {
|
|
16482
|
+
return;
|
|
16483
|
+
}
|
|
16484
|
+
if (subscribed) {
|
|
16485
|
+
const pendingOpeningCapabilities = this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
16486
|
+
if (pendingOpeningCapabilities &&
|
|
16487
|
+
pendingOpeningCapabilities.epoch !== expectedSubscriptionEpoch) {
|
|
16488
|
+
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
16489
|
+
}
|
|
16490
|
+
this._subscriptionOpeningEpochByPeer.set(peerHash, expectedSubscriptionEpoch);
|
|
16491
|
+
// Fence new messages immediately, drain handlers admitted by the previous
|
|
16492
|
+
// subscription, then wait behind every queued replication mutation. A
|
|
16493
|
+
// reconnect must not inherit metadata or ranges from the old connection.
|
|
16494
|
+
try {
|
|
16495
|
+
this._replicationInfoBlockedPeers.add(peerHash);
|
|
16496
|
+
await this.drainPeerReceiveHandlers(peerHash);
|
|
16497
|
+
await this.withReplicationInfoApplyQueue(peerHash, async () => { });
|
|
16498
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
16499
|
+
!ownsSubscriptionEpoch()) {
|
|
16500
|
+
return;
|
|
16501
|
+
}
|
|
16502
|
+
// The timestamp watermark belongs to the previous subscription epoch.
|
|
16503
|
+
// Sender clocks are not synchronized, so carrying a local unsubscribe
|
|
16504
|
+
// timestamp forward could reject every valid announcement after reconnect.
|
|
16505
|
+
this.latestReplicationInfoMessage.delete(peerHash);
|
|
16506
|
+
this._pendingReplicatorLeaveByPeer.delete(peerHash);
|
|
16507
|
+
const openingCapabilities = this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
16508
|
+
if (openingCapabilities?.epoch === expectedSubscriptionEpoch) {
|
|
16509
|
+
this._peerSyncCapabilities.set(peerHash, openingCapabilities.capabilities);
|
|
16510
|
+
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
16511
|
+
}
|
|
16512
|
+
this._replicationInfoBlockedPeers.delete(peerHash);
|
|
16513
|
+
}
|
|
16514
|
+
finally {
|
|
16515
|
+
if (this._openingSyncCapabilitiesByPeer.get(peerHash)?.epoch ===
|
|
16516
|
+
expectedSubscriptionEpoch) {
|
|
16517
|
+
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
16518
|
+
}
|
|
16519
|
+
if (this._subscriptionOpeningEpochByPeer.get(peerHash) ===
|
|
16520
|
+
expectedSubscriptionEpoch) {
|
|
16521
|
+
this._subscriptionOpeningEpochByPeer.delete(peerHash);
|
|
16522
|
+
}
|
|
16523
|
+
}
|
|
16524
|
+
}
|
|
15131
16525
|
if (!subscribed) {
|
|
16526
|
+
this._subscriptionOpeningEpochByPeer.delete(peerHash);
|
|
16527
|
+
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
15132
16528
|
this._replicationInfoBlockedPeers.add(peerHash);
|
|
16529
|
+
const disconnectedJoinWarmupGeneration = this._joinWarmupGenerationByTarget.get(peerHash) ?? null;
|
|
16530
|
+
this.cancelJoinWarmupTarget(peerHash);
|
|
15133
16531
|
const now = BigInt(+new Date());
|
|
15134
16532
|
const previous = this.latestReplicationInfoMessage.get(peerHash);
|
|
15135
16533
|
if (!previous || previous < now) {
|
|
15136
16534
|
this.latestReplicationInfoMessage.set(peerHash, now);
|
|
15137
16535
|
}
|
|
15138
|
-
|
|
16536
|
+
let removed = false;
|
|
15139
16537
|
try {
|
|
15140
16538
|
// Unsubscribe can race with the peer's final replication reset message.
|
|
15141
16539
|
// Proactively evict its ranges so leader selection doesn't keep stale owners.
|
|
15142
|
-
await this.removeReplicator(publicKey, {
|
|
16540
|
+
removed = await this.removeReplicator(publicKey, {
|
|
16541
|
+
cleanupIfSubscriptionSuperseded: true,
|
|
16542
|
+
expectedJoinWarmupGeneration: disconnectedJoinWarmupGeneration,
|
|
16543
|
+
noEvent: true,
|
|
16544
|
+
onRemoved: ({ wasReplicator }) => {
|
|
16545
|
+
if (wasReplicator) {
|
|
16546
|
+
this._pendingReplicatorLeaveByPeer.add(peerHash);
|
|
16547
|
+
}
|
|
16548
|
+
},
|
|
16549
|
+
replicationLifecycleController,
|
|
16550
|
+
subscriptionEpoch: expectedSubscriptionEpoch,
|
|
16551
|
+
});
|
|
15143
16552
|
}
|
|
15144
16553
|
catch (error) {
|
|
15145
16554
|
if (!isNotStartedError(error)) {
|
|
15146
16555
|
throw error;
|
|
15147
16556
|
}
|
|
15148
16557
|
}
|
|
15149
|
-
if (!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
16558
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
16559
|
+
!ownsSubscriptionEpoch() ||
|
|
16560
|
+
!removed) {
|
|
15150
16561
|
return;
|
|
15151
16562
|
}
|
|
15152
|
-
this.
|
|
15153
|
-
this.cleanupPeerDisconnectTracking(peerHash);
|
|
15154
|
-
if (wasReplicator) {
|
|
16563
|
+
if (this._pendingReplicatorLeaveByPeer.delete(peerHash)) {
|
|
15155
16564
|
this.events.dispatchEvent(new CustomEvent("replicator:leave", {
|
|
15156
16565
|
detail: { publicKey },
|
|
15157
16566
|
}));
|
|
@@ -15185,7 +16594,8 @@ let SharedLog = (() => {
|
|
|
15185
16594
|
}
|
|
15186
16595
|
throw error;
|
|
15187
16596
|
}
|
|
15188
|
-
if (!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
16597
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
16598
|
+
!ownsSubscriptionEpoch()) {
|
|
15189
16599
|
return;
|
|
15190
16600
|
}
|
|
15191
16601
|
if (replicationSegments.length > 0) {
|
|
@@ -15197,7 +16607,8 @@ let SharedLog = (() => {
|
|
|
15197
16607
|
signal: replicationLifecycleController.signal,
|
|
15198
16608
|
})
|
|
15199
16609
|
.catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
|
|
15200
|
-
if (!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
16610
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
16611
|
+
!ownsSubscriptionEpoch()) {
|
|
15201
16612
|
return;
|
|
15202
16613
|
}
|
|
15203
16614
|
if (this.v8Behaviour) {
|
|
@@ -15218,7 +16629,8 @@ let SharedLog = (() => {
|
|
|
15218
16629
|
// Request the remote peer's replication info. This makes joins resilient to
|
|
15219
16630
|
// timing-sensitive delivery/order issues where we may miss their initial
|
|
15220
16631
|
// replication announcement.
|
|
15221
|
-
if (this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
16632
|
+
if (this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
16633
|
+
ownsSubscriptionEpoch()) {
|
|
15222
16634
|
this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
|
|
15223
16635
|
}
|
|
15224
16636
|
}
|
|
@@ -15532,12 +16944,18 @@ let SharedLog = (() => {
|
|
|
15532
16944
|
if (this.closed) {
|
|
15533
16945
|
return;
|
|
15534
16946
|
}
|
|
15535
|
-
await this.log.trim();
|
|
15536
16947
|
const batchedChanges = Array.isArray(changeOrChanges[0])
|
|
15537
16948
|
? changeOrChanges
|
|
15538
16949
|
: [changeOrChanges];
|
|
15539
16950
|
const changes = batchedChanges.flat();
|
|
15540
16951
|
const selfHash = this.node.identity.publicKey.hashcode();
|
|
16952
|
+
const joinWarmupGenerations = new Map();
|
|
16953
|
+
for (const change of changes) {
|
|
16954
|
+
if (change.type === "added" && change.range.hash !== selfHash) {
|
|
16955
|
+
joinWarmupGenerations.set(change.range.hash, this.getJoinWarmupGeneration(change.range.hash));
|
|
16956
|
+
}
|
|
16957
|
+
}
|
|
16958
|
+
await this.log.trim();
|
|
15541
16959
|
// On removed ranges (peer leaves / shrink), gid-level history can hide
|
|
15542
16960
|
// per-entry gaps. Force a fresh delivery pass for reassigned entries.
|
|
15543
16961
|
const forceFreshDelivery = changes.some((change) => change.type === "removed");
|
|
@@ -15597,6 +17015,10 @@ let SharedLog = (() => {
|
|
|
15597
17015
|
? changes.filter((change) => !(change.range.hash === selfHash &&
|
|
15598
17016
|
(change.type === "added" || change.type === "replaced")))
|
|
15599
17017
|
: changes;
|
|
17018
|
+
const isCurrentJoinWarmupTarget = (target) => warmupPeers.has(target) &&
|
|
17019
|
+
this._joinWarmupGenerationByTarget.get(target) ===
|
|
17020
|
+
joinWarmupGenerations.get(target);
|
|
17021
|
+
const areJoinWarmupGenerationsCurrent = () => [...warmupPeers].every(isCurrentJoinWarmupTarget);
|
|
15600
17022
|
try {
|
|
15601
17023
|
const uncheckedDeliver = new Map();
|
|
15602
17024
|
const flushUncheckedDeliverTarget = (target) => {
|
|
@@ -15605,6 +17027,10 @@ let SharedLog = (() => {
|
|
|
15605
17027
|
return;
|
|
15606
17028
|
}
|
|
15607
17029
|
const isWarmupTarget = warmupPeers.has(target);
|
|
17030
|
+
if (isWarmupTarget && !isCurrentJoinWarmupTarget(target)) {
|
|
17031
|
+
uncheckedDeliver.delete(target);
|
|
17032
|
+
return;
|
|
17033
|
+
}
|
|
15608
17034
|
const mode = forceFreshDelivery
|
|
15609
17035
|
? "churn"
|
|
15610
17036
|
: isWarmupTarget
|
|
@@ -15624,6 +17050,9 @@ let SharedLog = (() => {
|
|
|
15624
17050
|
uncheckedDeliver.delete(target);
|
|
15625
17051
|
};
|
|
15626
17052
|
const queueUncheckedDeliver = (target, entry) => {
|
|
17053
|
+
if (warmupPeers.has(target) && !isCurrentJoinWarmupTarget(target)) {
|
|
17054
|
+
return;
|
|
17055
|
+
}
|
|
15627
17056
|
churnRepairPeers.add(target);
|
|
15628
17057
|
let set = uncheckedDeliver.get(target);
|
|
15629
17058
|
if (!set) {
|
|
@@ -15642,7 +17071,9 @@ let SharedLog = (() => {
|
|
|
15642
17071
|
for await (const entryReplicated of toRebalance(immediateRebalanceChanges, this.entryCoordinatesIndex, this.recentlyRebalanced, {
|
|
15643
17072
|
forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
|
|
15644
17073
|
})) {
|
|
15645
|
-
if (this.closed
|
|
17074
|
+
if (this.closed ||
|
|
17075
|
+
(useJoinWarmupFastPath &&
|
|
17076
|
+
!areJoinWarmupGenerationsCurrent())) {
|
|
15646
17077
|
break;
|
|
15647
17078
|
}
|
|
15648
17079
|
if (useJoinWarmupFastPath) {
|
|
@@ -15659,7 +17090,9 @@ let SharedLog = (() => {
|
|
|
15659
17090
|
}
|
|
15660
17091
|
const candidatePeers = new Set([selfHash]);
|
|
15661
17092
|
for (const target of warmupPeers) {
|
|
15662
|
-
|
|
17093
|
+
if (isCurrentJoinWarmupTarget(target)) {
|
|
17094
|
+
candidatePeers.add(target);
|
|
17095
|
+
}
|
|
15663
17096
|
}
|
|
15664
17097
|
if (oldPeersSet) {
|
|
15665
17098
|
for (const oldPeer of oldPeersSet) {
|
|
@@ -15671,6 +17104,9 @@ let SharedLog = (() => {
|
|
|
15671
17104
|
candidates: candidatePeers,
|
|
15672
17105
|
persist: false,
|
|
15673
17106
|
});
|
|
17107
|
+
if (!areJoinWarmupGenerationsCurrent()) {
|
|
17108
|
+
continue;
|
|
17109
|
+
}
|
|
15674
17110
|
if (oldPeersSet) {
|
|
15675
17111
|
for (const oldPeer of oldPeersSet) {
|
|
15676
17112
|
if (!currentPeers.has(oldPeer)) {
|
|
@@ -15679,11 +17115,11 @@ let SharedLog = (() => {
|
|
|
15679
17115
|
}
|
|
15680
17116
|
}
|
|
15681
17117
|
for (const [peer] of currentPeers) {
|
|
15682
|
-
if (
|
|
15683
|
-
this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer);
|
|
17118
|
+
if (isCurrentJoinWarmupTarget(peer)) {
|
|
17119
|
+
this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer, joinWarmupGenerations.get(peer));
|
|
15684
17120
|
}
|
|
15685
17121
|
}
|
|
15686
|
-
const authoritativePeers = [...currentPeers.keys()].filter((peer) => !
|
|
17122
|
+
const authoritativePeers = [...currentPeers.keys()].filter((peer) => !isCurrentJoinWarmupTarget(peer) &&
|
|
15687
17123
|
!this.hasPendingRepairSweepOptimisticPeer(entryReplicated.gid, peer));
|
|
15688
17124
|
this.addPeersToGidPeerHistory(entryReplicated.gid, authoritativePeers, true);
|
|
15689
17125
|
if (!currentPeers.has(selfHash)) {
|
|
@@ -15730,8 +17166,8 @@ let SharedLog = (() => {
|
|
|
15730
17166
|
}
|
|
15731
17167
|
}
|
|
15732
17168
|
for (const [peer] of currentPeers) {
|
|
15733
|
-
if (
|
|
15734
|
-
this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer);
|
|
17169
|
+
if (isCurrentJoinWarmupTarget(peer)) {
|
|
17170
|
+
this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer, joinWarmupGenerations.get(peer));
|
|
15735
17171
|
}
|
|
15736
17172
|
}
|
|
15737
17173
|
const authoritativePeers = [...currentPeers.keys()].filter((peer) => !addedPeers.has(peer) &&
|
|
@@ -15771,6 +17207,7 @@ let SharedLog = (() => {
|
|
|
15771
17207
|
this.scheduleRepairSweep({
|
|
15772
17208
|
mode: "join-warmup",
|
|
15773
17209
|
peers,
|
|
17210
|
+
joinWarmupGenerations,
|
|
15774
17211
|
});
|
|
15775
17212
|
}, 250);
|
|
15776
17213
|
timer.unref?.();
|
|
@@ -15827,6 +17264,7 @@ let SharedLog = (() => {
|
|
|
15827
17264
|
return;
|
|
15828
17265
|
}
|
|
15829
17266
|
const fromHash = evt.detail.from.hashcode();
|
|
17267
|
+
const subscriptionEpoch = this.advanceSubscriptionEpoch(fromHash);
|
|
15830
17268
|
this._replicationInfoBlockedPeers.add(fromHash);
|
|
15831
17269
|
this._recentRepairDispatch.delete(fromHash);
|
|
15832
17270
|
// Keep a per-peer timestamp watermark when we observe an unsubscribe. This
|
|
@@ -15838,17 +17276,19 @@ let SharedLog = (() => {
|
|
|
15838
17276
|
this.latestReplicationInfoMessage.set(fromHash, now);
|
|
15839
17277
|
}
|
|
15840
17278
|
this.invalidateSharedLogTopicSubscribersCache();
|
|
15841
|
-
return this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, false);
|
|
17279
|
+
return this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, false, subscriptionEpoch);
|
|
15842
17280
|
}
|
|
15843
17281
|
async _onSubscription(evt) {
|
|
15844
17282
|
logger.trace(`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(evt.detail.topics.map((x) => x))}'`);
|
|
15845
17283
|
if (!evt.detail.topics.includes(this.topic)) {
|
|
15846
17284
|
return;
|
|
15847
17285
|
}
|
|
17286
|
+
const fromHash = evt.detail.from.hashcode();
|
|
17287
|
+
const subscriptionEpoch = this.advanceSubscriptionEpoch(fromHash);
|
|
15848
17288
|
this.remoteBlocks.onReachable(evt.detail.from);
|
|
15849
|
-
this._replicationInfoBlockedPeers.
|
|
17289
|
+
this._replicationInfoBlockedPeers.add(fromHash);
|
|
15850
17290
|
this.invalidateSharedLogTopicSubscribersCache();
|
|
15851
|
-
await this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, true);
|
|
17291
|
+
await this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, true, subscriptionEpoch);
|
|
15852
17292
|
}
|
|
15853
17293
|
async rebalanceParticipation() {
|
|
15854
17294
|
// update more participation rate to converge to the average expected rate or bounded by
|
|
@@ -15998,7 +17438,7 @@ let SharedLog = (() => {
|
|
|
15998
17438
|
const ih = this._pendingIHave.get(entry.hash);
|
|
15999
17439
|
if (ih) {
|
|
16000
17440
|
ih.clear();
|
|
16001
|
-
|
|
17441
|
+
this.runPendingIHaveCallback(ih, entry);
|
|
16002
17442
|
}
|
|
16003
17443
|
this.syncronizer.onEntryAdded(entry);
|
|
16004
17444
|
}
|
|
@@ -16010,7 +17450,7 @@ let SharedLog = (() => {
|
|
|
16010
17450
|
}
|
|
16011
17451
|
const entry = materializeEntry();
|
|
16012
17452
|
ih.clear();
|
|
16013
|
-
|
|
17453
|
+
this.runPendingIHaveCallback(ih, entry);
|
|
16014
17454
|
this.syncronizer.onEntryAdded(entry);
|
|
16015
17455
|
return;
|
|
16016
17456
|
}
|