@peerbit/shared-log 13.2.31 → 13.2.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/exchange-heads.d.ts +13 -2
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +13 -2
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +11 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +727 -85
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-announcement.d.ts +3 -1
- package/dist/src/replication-announcement.d.ts.map +1 -1
- package/dist/src/replication-announcement.js +1 -0
- package/dist/src/replication-announcement.js.map +1 -1
- package/dist/src/replication-info-v2-binding.d.ts +16 -0
- package/dist/src/replication-info-v2-binding.d.ts.map +1 -0
- package/dist/src/replication-info-v2-binding.js +30 -0
- package/dist/src/replication-info-v2-binding.js.map +1 -0
- package/dist/src/replication-info-v2-receive.d.ts +181 -0
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -0
- package/dist/src/replication-info-v2-receive.js +725 -0
- package/dist/src/replication-info-v2-receive.js.map +1 -0
- package/dist/src/replication-info-v2-send.d.ts +95 -0
- package/dist/src/replication-info-v2-send.d.ts.map +1 -0
- package/dist/src/replication-info-v2-send.js +398 -0
- package/dist/src/replication-info-v2-send.js.map +1 -0
- package/dist/src/replication.d.ts +26 -4
- package/dist/src/replication.d.ts.map +1 -1
- package/dist/src/replication.js +60 -5
- package/dist/src/replication.js.map +1 -1
- package/package.json +11 -11
- package/src/exchange-heads.ts +15 -2
- package/src/index.ts +1040 -201
- package/src/replication-announcement.ts +10 -4
- package/src/replication-info-v2-binding.ts +44 -0
- package/src/replication-info-v2-receive.ts +1045 -0
- package/src/replication-info-v2-send.ts +540 -0
- package/src/replication.ts +39 -5
package/dist/src/index.js
CHANGED
|
@@ -51,26 +51,28 @@ import PQueue from "p-queue";
|
|
|
51
51
|
import { concat, fromString } from "uint8arrays";
|
|
52
52
|
import { BlocksMessage } from "./blocks.js";
|
|
53
53
|
import { CheckedPruneCoordinator, } from "./checked-prune.js";
|
|
54
|
+
import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isPromiseLike, mapMaybePromise, normalizedHashValues, } from "./coordinate-persistence.js";
|
|
54
55
|
import { CPUUsageIntervalLag } from "./cpu.js";
|
|
55
56
|
import { debouncedAccumulatorMap, } from "./debounce.js";
|
|
56
57
|
import { NativeDurableCommitError, NoPeersError, isNotStartedError, } from "./errors.js";
|
|
57
|
-
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
58
|
+
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
58
59
|
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
59
|
-
import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isPromiseLike, mapMaybePromise, normalizedHashValues, } from "./coordinate-persistence.js";
|
|
60
60
|
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
61
61
|
import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
|
|
62
62
|
import { JoinWarmupCoordinator } from "./join-warmup.js";
|
|
63
63
|
import { LeaderPlanCache } from "./leader-plan-cache.js";
|
|
64
64
|
import { TransportMessage } from "./message.js";
|
|
65
65
|
import { NativeBackboneWriteThroughBlockStore } from "./native-write-through-block-store.js";
|
|
66
|
-
import { PeerSessionRegistry
|
|
66
|
+
import { PeerSessionRegistry } from "./peer-session.js";
|
|
67
67
|
import { PIDReplicationController } from "./pid.js";
|
|
68
68
|
import { EntryReplicatedU32, EntryReplicatedU64, ReplicationIntent, ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, ReplicationRangeMessage, appromixateCoverage, calculateCoverage, countCoveringRangesSameOwner, createAssignedRangesQuery, debounceAggregationChanges, getAllMergeCandiates, getCoverSet, getSamples, isEntryReplicated, isMatured, isReplicationRangeMessage, mergeRanges, minimumWidthToCover, shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary, toRebalance, } from "./ranges.js";
|
|
69
69
|
import { ReplicationAnnouncementCoordinator, isTransientReplicationAnnouncementError, } from "./replication-announcement.js";
|
|
70
70
|
import { createReplicationDomainHash, } from "./replication-domain-hash.js";
|
|
71
71
|
import { createReplicationDomainTime, } from "./replication-domain-time.js";
|
|
72
72
|
import {} from "./replication-domain.js";
|
|
73
|
-
import {
|
|
73
|
+
import { ReplicationInfoV2ReceiveCoordinator } from "./replication-info-v2-receive.js";
|
|
74
|
+
import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
|
|
75
|
+
import { AbsoluteReplicas, AddedReplicationInfoV2Message, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, FullReplicationInfoV2Message, MinReplicas, ReplicationError, ReplicationPingMessage, RequestReplicationInfoMessage, RequestReplicationInfoV2Message, ResponseRoleMessage, StoppedReplicating, StoppedReplicationInfoV2Message, decodeReplicas, encodeReplicas, isReplicationInfoV2Message, maxReplicas, } from "./replication.js";
|
|
74
76
|
import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
|
|
75
77
|
import { Observer, Replicator } from "./role.js";
|
|
76
78
|
import { createSyncronizer } from "./sync/factory.js";
|
|
@@ -624,7 +626,8 @@ let SharedLog = (() => {
|
|
|
624
626
|
value;
|
|
625
627
|
}
|
|
626
628
|
get _nativeBackboneCoordinateJournalLastFlushMs() {
|
|
627
|
-
return this._coordinates
|
|
629
|
+
return this._coordinates
|
|
630
|
+
?._nativeBackboneCoordinateJournalLastFlushMs;
|
|
628
631
|
}
|
|
629
632
|
set _nativeBackboneCoordinateJournalLastFlushMs(value) {
|
|
630
633
|
(this._coordinates ??=
|
|
@@ -1497,7 +1500,8 @@ let SharedLog = (() => {
|
|
|
1497
1500
|
}
|
|
1498
1501
|
if (!lowerMarkerCommitted) {
|
|
1499
1502
|
if (intent.coordinates.length > 0) {
|
|
1500
|
-
const mutationGenerations = (this._coordinates._nativeCoordinateMutationGenerations ??=
|
|
1503
|
+
const mutationGenerations = (this._coordinates._nativeCoordinateMutationGenerations ??=
|
|
1504
|
+
new Map());
|
|
1501
1505
|
const rollback = {
|
|
1502
1506
|
hashes: new Set(),
|
|
1503
1507
|
entries: new Map(),
|
|
@@ -1608,6 +1612,8 @@ let SharedLog = (() => {
|
|
|
1608
1612
|
// A fn that we can call many times that recalculates the participation role
|
|
1609
1613
|
rebalanceParticipationDebounced;
|
|
1610
1614
|
_announcements;
|
|
1615
|
+
_v2Receive;
|
|
1616
|
+
_v2Send;
|
|
1611
1617
|
// A fn for debouncing the calls for pruning
|
|
1612
1618
|
pruneDebouncedFn;
|
|
1613
1619
|
_checkedPruneAuditTimer;
|
|
@@ -1637,6 +1643,10 @@ let SharedLog = (() => {
|
|
|
1637
1643
|
// Sync capability bits advertised by peers (SyncCapabilitiesMessage), keyed
|
|
1638
1644
|
// by public key hash. Entries are dropped on unsubscribe/disconnect.
|
|
1639
1645
|
_peerSyncCapabilities;
|
|
1646
|
+
// Signed transport session carried by the capability envelope. Kept in a
|
|
1647
|
+
// parallel map so existing capability-number consumers remain unchanged.
|
|
1648
|
+
_peerSyncCapabilitySessions;
|
|
1649
|
+
_peerSyncCapabilityTimestamps;
|
|
1640
1650
|
// Pending live raw exchange-head gossip, coalesced per recipient set and
|
|
1641
1651
|
// flushed at the end of the current event-loop turn (or when a batch cap
|
|
1642
1652
|
// is hit). Only used when every recipient advertised raw capability.
|
|
@@ -1690,6 +1700,7 @@ let SharedLog = (() => {
|
|
|
1690
1700
|
// this).
|
|
1691
1701
|
queueCurrentReplicationStateAnnouncementRepair: () => this._announcements.queueCurrentReplicationStateAnnouncementRepair(),
|
|
1692
1702
|
queueCurrentReplicationStateAnnouncementRetry: (error) => this._announcements.queueCurrentReplicationStateAnnouncementRetry(error),
|
|
1703
|
+
enqueueReplicationInfoV2: (message) => this._v2Send.enqueue(message),
|
|
1693
1704
|
isClosed: () => this.closed,
|
|
1694
1705
|
getCloseSignal: () => this._closeController.signal,
|
|
1695
1706
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
@@ -1704,6 +1715,94 @@ let SharedLog = (() => {
|
|
|
1704
1715
|
callRebalanceParticipationDebounced: () => this.rebalanceParticipationDebounced?.call(),
|
|
1705
1716
|
});
|
|
1706
1717
|
}
|
|
1718
|
+
createReplicationInfoV2SendCoordinator() {
|
|
1719
|
+
return new ReplicationInfoV2SendCoordinator({
|
|
1720
|
+
getRpc: () => this.rpc,
|
|
1721
|
+
getSelfKey: () => this.node.identity.publicKey,
|
|
1722
|
+
getSenderTransportSession: () => BigInt(this.node.services.pubsub.session),
|
|
1723
|
+
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
1724
|
+
validatePersistedReplicationRangeSnapshot: (ranges) => this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
1725
|
+
isClosed: () => this.closed,
|
|
1726
|
+
isPeerSessionOpen: (peerHash, peerSession) => this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
1727
|
+
peerSession.phase === "open" &&
|
|
1728
|
+
!this._peerSessions.isReplicationInfoBlocked(peerHash) &&
|
|
1729
|
+
this._peerSessions.isReceiveCleanupGateOpen(peerHash),
|
|
1730
|
+
captureReplicationOwnershipLifecycle: () => this.captureReplicationOwnershipLifecycle(),
|
|
1731
|
+
isReplicationOwnershipLifecycleActive: (controller) => this.isRepairLifecycleActive(controller),
|
|
1732
|
+
});
|
|
1733
|
+
}
|
|
1734
|
+
replicationInfoV2ReceiveCapabilities() {
|
|
1735
|
+
return (SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
1736
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
|
|
1737
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
|
|
1738
|
+
(this._logProperties?.sync?.rawExchangeHeads === true
|
|
1739
|
+
? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
|
|
1740
|
+
: 0));
|
|
1741
|
+
}
|
|
1742
|
+
async advertiseReplicationInfoV2ReceiveCapability(properties) {
|
|
1743
|
+
const peerHash = properties.target.hashcode();
|
|
1744
|
+
const receiverTransportSession = BigInt(this.node.services.pubsub.session);
|
|
1745
|
+
await this.rpc.send(new SyncCapabilitiesMessage({
|
|
1746
|
+
capabilities: this.replicationInfoV2ReceiveCapabilities(),
|
|
1747
|
+
}), {
|
|
1748
|
+
mode: new AcknowledgeDelivery({
|
|
1749
|
+
redundancy: 1,
|
|
1750
|
+
to: [properties.target],
|
|
1751
|
+
}),
|
|
1752
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
1753
|
+
signal: properties.signal,
|
|
1754
|
+
});
|
|
1755
|
+
if (properties.signal.aborted ||
|
|
1756
|
+
this.closed ||
|
|
1757
|
+
!this._peerSessions.isCurrent(peerHash, properties.peerSession) ||
|
|
1758
|
+
properties.peerSession.phase !== "open" ||
|
|
1759
|
+
!properties.peerSession.isActive() ||
|
|
1760
|
+
!this._peerSessions.isReceiveEpochCurrent(peerHash, properties.receiveEpoch) ||
|
|
1761
|
+
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
1762
|
+
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
1763
|
+
BigInt(this.node.services.pubsub.session) !== receiverTransportSession) {
|
|
1764
|
+
return undefined;
|
|
1765
|
+
}
|
|
1766
|
+
return {
|
|
1767
|
+
receiverTransportSession,
|
|
1768
|
+
requestNotBeforeMs: Date.now(),
|
|
1769
|
+
};
|
|
1770
|
+
}
|
|
1771
|
+
createReplicationInfoV2ReceiveCoordinator() {
|
|
1772
|
+
return new ReplicationInfoV2ReceiveCoordinator({
|
|
1773
|
+
getSelfKey: () => this.node.identity.publicKey,
|
|
1774
|
+
getReceiverTransportSession: () => BigInt(this.node.services.pubsub.session),
|
|
1775
|
+
isClosed: () => this.closed,
|
|
1776
|
+
isPeerStateCurrent: (peerHash, peerSession, receiveEpoch) => this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
1777
|
+
peerSession.phase === "open" &&
|
|
1778
|
+
peerSession.isActive() &&
|
|
1779
|
+
this._peerSessions.isReceiveEpochCurrent(peerHash, receiveEpoch) &&
|
|
1780
|
+
!this._peerSessions.isReplicationInfoBlocked(peerHash) &&
|
|
1781
|
+
this._peerSessions.isReceiveCleanupGateOpen(peerHash),
|
|
1782
|
+
isSenderTransportSessionCurrent: (peerHash, senderTransportSession) => this._peerSyncCapabilitySessions.get(peerHash) ===
|
|
1783
|
+
senderTransportSession,
|
|
1784
|
+
sendRequest: async (request, target, signal) => {
|
|
1785
|
+
await this.rpc.send(request, {
|
|
1786
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [target] }),
|
|
1787
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
1788
|
+
signal,
|
|
1789
|
+
});
|
|
1790
|
+
},
|
|
1791
|
+
refreshLocalCapability: (properties) => this.advertiseReplicationInfoV2ReceiveCapability({
|
|
1792
|
+
target: properties.target,
|
|
1793
|
+
peerSession: properties.peerSession,
|
|
1794
|
+
receiveEpoch: properties.receiveEpoch,
|
|
1795
|
+
signal: properties.signal,
|
|
1796
|
+
}),
|
|
1797
|
+
onRequestError: (error) => {
|
|
1798
|
+
if (isNotStartedError(error) ||
|
|
1799
|
+
(this.closed && error instanceof AbortError)) {
|
|
1800
|
+
return;
|
|
1801
|
+
}
|
|
1802
|
+
logger.trace(`Replication-info V2 recovery request failed: ${error?.message ?? String(error)}`);
|
|
1803
|
+
},
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1707
1806
|
createReplicatorLivenessMonitor() {
|
|
1708
1807
|
return new ReplicatorLivenessMonitor({
|
|
1709
1808
|
// Sweep-driven probes and activity marks dispatch through the owner so
|
|
@@ -1798,6 +1897,8 @@ let SharedLog = (() => {
|
|
|
1798
1897
|
this._appendBackfillPendingByTarget = new Map();
|
|
1799
1898
|
this._topicSubscribersCache = new Map();
|
|
1800
1899
|
this._peerSyncCapabilities = new Map();
|
|
1900
|
+
this._peerSyncCapabilitySessions = new Map();
|
|
1901
|
+
this._peerSyncCapabilityTimestamps = new Map();
|
|
1801
1902
|
this._liveRawGossipBatches = new Map();
|
|
1802
1903
|
this._liveRawGossipFlushScheduled = false;
|
|
1803
1904
|
this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
|
|
@@ -1806,6 +1907,8 @@ let SharedLog = (() => {
|
|
|
1806
1907
|
this._replicatorJoinEmitted = new Set();
|
|
1807
1908
|
this._replicatorsReconciled = false;
|
|
1808
1909
|
this._liveness = this.createReplicatorLivenessMonitor();
|
|
1910
|
+
this._v2Receive = this.createReplicationInfoV2ReceiveCoordinator();
|
|
1911
|
+
this._v2Send = this.createReplicationInfoV2SendCoordinator();
|
|
1809
1912
|
this._announcements = this.createReplicationAnnouncementCoordinator();
|
|
1810
1913
|
this.pendingMaturity = new Map();
|
|
1811
1914
|
this._closeController = new AbortController();
|
|
@@ -2196,6 +2299,101 @@ let SharedLog = (() => {
|
|
|
2196
2299
|
SYNC_CAPABILITY_RAW_EXCHANGE_HEADS) !==
|
|
2197
2300
|
0);
|
|
2198
2301
|
}
|
|
2302
|
+
observePeerSyncCapabilities(properties) {
|
|
2303
|
+
const { peerHash, capabilities, transportSession, timestamp, openingSession, } = properties;
|
|
2304
|
+
if (openingSession) {
|
|
2305
|
+
const previous = this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
2306
|
+
if (transportSession !== undefined &&
|
|
2307
|
+
timestamp !== undefined &&
|
|
2308
|
+
previous?.epoch === openingSession &&
|
|
2309
|
+
previous.timestamp !== undefined &&
|
|
2310
|
+
previous.transportSession !== transportSession &&
|
|
2311
|
+
timestamp <= previous.timestamp) {
|
|
2312
|
+
return false;
|
|
2313
|
+
}
|
|
2314
|
+
if (transportSession !== undefined &&
|
|
2315
|
+
timestamp !== undefined &&
|
|
2316
|
+
previous?.epoch === openingSession &&
|
|
2317
|
+
previous.transportSession === transportSession) {
|
|
2318
|
+
if (previous.timestamp !== undefined &&
|
|
2319
|
+
timestamp < previous.timestamp) {
|
|
2320
|
+
return false;
|
|
2321
|
+
}
|
|
2322
|
+
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
2323
|
+
epoch: openingSession,
|
|
2324
|
+
capabilities: previous.capabilities | capabilities,
|
|
2325
|
+
transportSession,
|
|
2326
|
+
timestamp: previous.timestamp === undefined || timestamp > previous.timestamp
|
|
2327
|
+
? timestamp
|
|
2328
|
+
: previous.timestamp,
|
|
2329
|
+
});
|
|
2330
|
+
return true;
|
|
2331
|
+
}
|
|
2332
|
+
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
2333
|
+
epoch: openingSession,
|
|
2334
|
+
capabilities,
|
|
2335
|
+
transportSession,
|
|
2336
|
+
timestamp,
|
|
2337
|
+
});
|
|
2338
|
+
return true;
|
|
2339
|
+
}
|
|
2340
|
+
if (transportSession === undefined || timestamp === undefined) {
|
|
2341
|
+
// Test/in-process synthetic contexts predate signed envelope captures.
|
|
2342
|
+
// They may exercise capability-number behavior, but can never authorize V2.
|
|
2343
|
+
this._peerSyncCapabilities.set(peerHash, capabilities);
|
|
2344
|
+
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
2345
|
+
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
2346
|
+
this._v2Send.advancePeerCapability(peerHash);
|
|
2347
|
+
this._v2Receive.revokePeerCapability(peerHash);
|
|
2348
|
+
return true;
|
|
2349
|
+
}
|
|
2350
|
+
const previousSession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
2351
|
+
const previousTimestamp = this._peerSyncCapabilityTimestamps.get(peerHash);
|
|
2352
|
+
const sameTransportSession = previousSession === transportSession;
|
|
2353
|
+
if (previousTimestamp !== undefined &&
|
|
2354
|
+
((sameTransportSession && timestamp < previousTimestamp) ||
|
|
2355
|
+
(!sameTransportSession && timestamp <= previousTimestamp))) {
|
|
2356
|
+
return false;
|
|
2357
|
+
}
|
|
2358
|
+
const previousCapabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
|
|
2359
|
+
const nextCapabilities = sameTransportSession
|
|
2360
|
+
? previousCapabilities | capabilities
|
|
2361
|
+
: capabilities;
|
|
2362
|
+
const senderGrantCapabilityMask = SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
2363
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY;
|
|
2364
|
+
const generationAdvanced = !sameTransportSession ||
|
|
2365
|
+
(previousCapabilities & senderGrantCapabilityMask) !==
|
|
2366
|
+
(nextCapabilities & senderGrantCapabilityMask);
|
|
2367
|
+
this._peerSyncCapabilities.set(peerHash, nextCapabilities);
|
|
2368
|
+
this._peerSyncCapabilitySessions.set(peerHash, transportSession);
|
|
2369
|
+
this._peerSyncCapabilityTimestamps.set(peerHash, previousTimestamp === undefined ||
|
|
2370
|
+
!sameTransportSession ||
|
|
2371
|
+
timestamp > previousTimestamp
|
|
2372
|
+
? timestamp
|
|
2373
|
+
: previousTimestamp);
|
|
2374
|
+
if (generationAdvanced) {
|
|
2375
|
+
this._v2Send.advancePeerCapability(peerHash);
|
|
2376
|
+
}
|
|
2377
|
+
return true;
|
|
2378
|
+
}
|
|
2379
|
+
promoteReplicationInfoV2ReceiveCapability(target, peerSession) {
|
|
2380
|
+
const peerHash = target.hashcode();
|
|
2381
|
+
const senderTransportSession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
2382
|
+
const capabilityTimestamp = this._peerSyncCapabilityTimestamps.get(peerHash);
|
|
2383
|
+
if (senderTransportSession === undefined ||
|
|
2384
|
+
capabilityTimestamp === undefined) {
|
|
2385
|
+
return false;
|
|
2386
|
+
}
|
|
2387
|
+
return this._v2Receive.observeCapability({
|
|
2388
|
+
peerHash,
|
|
2389
|
+
target,
|
|
2390
|
+
peerSession,
|
|
2391
|
+
receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
|
|
2392
|
+
capabilities: this._peerSyncCapabilities.get(peerHash) ?? 0,
|
|
2393
|
+
senderTransportSession,
|
|
2394
|
+
capabilityTimestamp,
|
|
2395
|
+
});
|
|
2396
|
+
}
|
|
2199
2397
|
/**
|
|
2200
2398
|
* Live append gossip may use the raw exchange-heads path only when we
|
|
2201
2399
|
* opted into raw sync and every remote recipient advertised raw capability
|
|
@@ -2892,7 +3090,8 @@ let SharedLog = (() => {
|
|
|
2892
3090
|
return undefined;
|
|
2893
3091
|
}
|
|
2894
3092
|
isReceiveOwnershipSnapshotStable(revision) {
|
|
2895
|
-
return ((this._instanceLifecycle?._receiveOwnershipMutationAdmissions ?? 0) ===
|
|
3093
|
+
return ((this._instanceLifecycle?._receiveOwnershipMutationAdmissions ?? 0) ===
|
|
3094
|
+
0 &&
|
|
2896
3095
|
(this._instanceLifecycle?._receiveOwnershipRevision ?? 0) === revision);
|
|
2897
3096
|
}
|
|
2898
3097
|
canCacheLeaderSelectionContext(options) {
|
|
@@ -3679,6 +3878,7 @@ let SharedLog = (() => {
|
|
|
3679
3878
|
};
|
|
3680
3879
|
let removed = false;
|
|
3681
3880
|
let removalCallCompleted = false;
|
|
3881
|
+
let replicationInfoRecoveryEpochAdvanced = false;
|
|
3682
3882
|
// Replication-info updates already serialize per peer. Put the hash-wide
|
|
3683
3883
|
// removal on the same queue so a newer reset cannot be deleted underneath
|
|
3684
3884
|
// itself by an older unsubscribe callback.
|
|
@@ -3748,6 +3948,13 @@ let SharedLog = (() => {
|
|
|
3748
3948
|
// before mutating ownership.
|
|
3749
3949
|
this.cleanupCheckedPrunePeer(keyHash, replicationOwnershipLifecycleController, checkedPruneCoordinator);
|
|
3750
3950
|
}
|
|
3951
|
+
if (!isMe) {
|
|
3952
|
+
// This is the last synchronous current-generation boundary
|
|
3953
|
+
// before destructive work. Fence parked receives now because
|
|
3954
|
+
// the coherent delete can durably mutate and then fail or poison.
|
|
3955
|
+
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
3956
|
+
replicationInfoRecoveryEpochAdvanced = true;
|
|
3957
|
+
}
|
|
3751
3958
|
const deletion = await this.deleteReplicationRangesCoherently(deleted, keyHash);
|
|
3752
3959
|
deleted = deletion.removed;
|
|
3753
3960
|
ownerHasRanges = deletion.ownerHasRanges;
|
|
@@ -3793,11 +4000,6 @@ let SharedLog = (() => {
|
|
|
3793
4000
|
// through replication-info updates without a topic unsubscribe event.
|
|
3794
4001
|
await cleanupDisconnectedPeer();
|
|
3795
4002
|
if (!isMe) {
|
|
3796
|
-
// Replication-info handlers release their receive lease before joining
|
|
3797
|
-
// this lane. Fence every handler queued behind this successful removal,
|
|
3798
|
-
// regardless of whether it came from liveness, startup pruning, or an
|
|
3799
|
-
// unsubscribe transition.
|
|
3800
|
-
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
3801
4003
|
this.rebalanceParticipationDebounced?.call();
|
|
3802
4004
|
}
|
|
3803
4005
|
removed = true;
|
|
@@ -3828,6 +4030,22 @@ let SharedLog = (() => {
|
|
|
3828
4030
|
}
|
|
3829
4031
|
finally {
|
|
3830
4032
|
releaseReceiveCleanupGate?.();
|
|
4033
|
+
if (replicationInfoRecoveryEpochAdvanced &&
|
|
4034
|
+
ownsReplicationOwnershipLifecycle() &&
|
|
4035
|
+
ownsReplicationLifecycle() &&
|
|
4036
|
+
ownsSubscriptionEpoch()) {
|
|
4037
|
+
// The first recovery arm ran while receive admission was fenced.
|
|
4038
|
+
// Re-arm after releasing the cleanup gate so a still-open peer can
|
|
4039
|
+
// deliver the authoritative Full that completes recovery.
|
|
4040
|
+
const peerSession = this._peerSessions.current(keyHash);
|
|
4041
|
+
if (peerSession?.phase === "open") {
|
|
4042
|
+
this._v2Receive.advanceRecovery({
|
|
4043
|
+
peerHash: keyHash,
|
|
4044
|
+
peerSession,
|
|
4045
|
+
receiveEpoch: this._peerSessions.receiveEpoch(keyHash),
|
|
4046
|
+
});
|
|
4047
|
+
}
|
|
4048
|
+
}
|
|
3831
4049
|
if (isSpeculativePeerRemoval) {
|
|
3832
4050
|
const cancelledByFreshActivity = removalCallCompleted &&
|
|
3833
4051
|
!removed &&
|
|
@@ -3920,6 +4138,11 @@ let SharedLog = (() => {
|
|
|
3920
4138
|
}
|
|
3921
4139
|
const deletion = await this.deleteReplicationRangesCoherently(ranges, ownerHash);
|
|
3922
4140
|
ranges = deletion.removed;
|
|
4141
|
+
if ((options?.shouldRemove && !options.shouldRemove()) ||
|
|
4142
|
+
options?.onDurableRemoveCommitted?.() === false) {
|
|
4143
|
+
await deletion.rollback();
|
|
4144
|
+
return false;
|
|
4145
|
+
}
|
|
3923
4146
|
options?.onRemoved?.(ranges);
|
|
3924
4147
|
if (ranges.length > 0) {
|
|
3925
4148
|
this.events.dispatchEvent(new CustomEvent("replication:change", {
|
|
@@ -3983,7 +4206,7 @@ let SharedLog = (() => {
|
|
|
3983
4206
|
this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
|
|
3984
4207
|
return this.withReceiveOwnershipMutationQueue(() => this.addReplicationRangeUnlocked(ranges, from, options, replicationOwnershipLifecycleController), replicationOwnershipLifecycleController);
|
|
3985
4208
|
}
|
|
3986
|
-
async addReplicationRangeUnlocked(ranges, from, { reset, checkDuplicates, timestamp: ts, rebalance, allowLegacyOrderedReplacementPairs, onConfirmedDurableStateChanged, shouldApply, } = {}, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
|
|
4209
|
+
async addReplicationRangeUnlocked(ranges, from, { reset, checkDuplicates, timestamp: ts, rebalance, allowLegacyOrderedReplacementPairs, onConfirmedDurableStateChanged, onDurableApplyCommitted, shouldApply, } = {}, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
|
|
3987
4210
|
this.validateReplicationRangeAnnouncement(ranges);
|
|
3988
4211
|
this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
|
|
3989
4212
|
// This predicate is intentionally checked inside the global ownership lane.
|
|
@@ -3992,6 +4215,12 @@ let SharedLog = (() => {
|
|
|
3992
4215
|
if (shouldApply && !shouldApply()) {
|
|
3993
4216
|
return [];
|
|
3994
4217
|
}
|
|
4218
|
+
const applySuperseded = Symbol("replication-range-apply-superseded");
|
|
4219
|
+
const throwIfApplySuperseded = () => {
|
|
4220
|
+
if (shouldApply && !shouldApply()) {
|
|
4221
|
+
throw applySuperseded;
|
|
4222
|
+
}
|
|
4223
|
+
};
|
|
3995
4224
|
const fromHash = from.hashcode();
|
|
3996
4225
|
const incomingRangesById = new Map();
|
|
3997
4226
|
const incomingRangeCountsById = new Map();
|
|
@@ -4050,6 +4279,7 @@ let SharedLog = (() => {
|
|
|
4050
4279
|
let isStoppedReplicating = false;
|
|
4051
4280
|
let wasReplicatorBeforeDestructiveReset = false;
|
|
4052
4281
|
let resetFailureLeaveEmitted = false;
|
|
4282
|
+
let resetDeletionRollback;
|
|
4053
4283
|
const publishConfirmedResetStop = (ownerHasRanges) => {
|
|
4054
4284
|
if (ownerHasRanges || resetFailureLeaveEmitted) {
|
|
4055
4285
|
return;
|
|
@@ -4090,8 +4320,14 @@ let SharedLog = (() => {
|
|
|
4090
4320
|
else {
|
|
4091
4321
|
wasReplicatorBeforeDestructiveReset =
|
|
4092
4322
|
this.uniqueReplicators.has(fromHash);
|
|
4323
|
+
throwIfApplySuperseded();
|
|
4093
4324
|
const deletion = await this.deleteReplicationRangesCoherently(deleted, fromHash, { preserveOwnerMembership: ranges.length > 0 });
|
|
4094
4325
|
deleted = deletion.removed;
|
|
4326
|
+
resetDeletionRollback = deletion.rollback;
|
|
4327
|
+
if (shouldApply && !shouldApply()) {
|
|
4328
|
+
await deletion.rollback();
|
|
4329
|
+
return [];
|
|
4330
|
+
}
|
|
4095
4331
|
diffs = [
|
|
4096
4332
|
...deleted.map((x) => {
|
|
4097
4333
|
return { range: x, type: "removed", timestamp };
|
|
@@ -4247,19 +4483,27 @@ let SharedLog = (() => {
|
|
|
4247
4483
|
return failure;
|
|
4248
4484
|
};
|
|
4249
4485
|
try {
|
|
4486
|
+
throwIfApplySuperseded();
|
|
4250
4487
|
for (const diff of diffs) {
|
|
4251
4488
|
if (diff.type !== "added") {
|
|
4252
4489
|
continue;
|
|
4253
4490
|
}
|
|
4491
|
+
throwIfApplySuperseded();
|
|
4254
4492
|
appliedPositiveRanges.push(diff);
|
|
4255
4493
|
await this.replicationIndex.put(diff.range);
|
|
4256
4494
|
this.putNativeReplicationRange(diff.range);
|
|
4495
|
+
throwIfApplySuperseded();
|
|
4257
4496
|
}
|
|
4258
4497
|
if (reset && diffs.length > 0) {
|
|
4259
4498
|
await this.updateOldestTimestampFromIndex();
|
|
4499
|
+
throwIfApplySuperseded();
|
|
4500
|
+
}
|
|
4501
|
+
if (onDurableApplyCommitted?.() === false) {
|
|
4502
|
+
throw applySuperseded;
|
|
4260
4503
|
}
|
|
4261
4504
|
}
|
|
4262
4505
|
catch (error) {
|
|
4506
|
+
const superseded = error === applySuperseded;
|
|
4263
4507
|
let outcomeError = error;
|
|
4264
4508
|
if (appliedPositiveRanges.length > 0) {
|
|
4265
4509
|
try {
|
|
@@ -4269,6 +4513,20 @@ let SharedLog = (() => {
|
|
|
4269
4513
|
outcomeError = poisonFromPositiveRollback(rollbackError, error);
|
|
4270
4514
|
}
|
|
4271
4515
|
}
|
|
4516
|
+
if (superseded) {
|
|
4517
|
+
if (resetDeletionRollback) {
|
|
4518
|
+
try {
|
|
4519
|
+
await resetDeletionRollback();
|
|
4520
|
+
}
|
|
4521
|
+
catch (rollbackError) {
|
|
4522
|
+
outcomeError = rollbackError;
|
|
4523
|
+
}
|
|
4524
|
+
}
|
|
4525
|
+
if (outcomeError === applySuperseded) {
|
|
4526
|
+
return [];
|
|
4527
|
+
}
|
|
4528
|
+
throw outcomeError;
|
|
4529
|
+
}
|
|
4272
4530
|
if (reset) {
|
|
4273
4531
|
const negativeDiffs = diffs.filter((diff) => diff.type !== "added");
|
|
4274
4532
|
if (negativeDiffs.length > 0) {
|
|
@@ -6875,7 +7133,8 @@ let SharedLog = (() => {
|
|
|
6875
7133
|
appendLocallyPreparedPayloadNativeBackboneStorageTransaction(payloadData, appendOptions, properties, minReplicasValue, runtimeOnlyCoordinates, ownershipLifecycleController) {
|
|
6876
7134
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
6877
7135
|
const backbone = this._nativeBackbone;
|
|
6878
|
-
if (!backbone ||
|
|
7136
|
+
if (!backbone ||
|
|
7137
|
+
!this._coordinates.canUseNativeBackboneResidentCoordinateState()) {
|
|
6879
7138
|
return undefined;
|
|
6880
7139
|
}
|
|
6881
7140
|
if (properties?.nativeBackboneDocumentDeleteKey &&
|
|
@@ -7006,11 +7265,12 @@ let SharedLog = (() => {
|
|
|
7006
7265
|
backboneAppend.trimmed.map((entry) => entry.hash);
|
|
7007
7266
|
const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
7008
7267
|
const committedNext = backboneAppend.entry.next ?? next;
|
|
7009
|
-
nativeCoordinateRollback =
|
|
7010
|
-
|
|
7011
|
-
|
|
7012
|
-
|
|
7013
|
-
|
|
7268
|
+
nativeCoordinateRollback =
|
|
7269
|
+
this._coordinates.snapshotResidentCoordinateEntries([
|
|
7270
|
+
...(committedHash ? [committedHash] : []),
|
|
7271
|
+
...committedNext,
|
|
7272
|
+
...nativeTrimmedHashes,
|
|
7273
|
+
]);
|
|
7014
7274
|
await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, committedHash ? [committedHash] : [], nativeTrimmedHashes, nativeCoordinateRollback, combineCoordinateDeleteHashes(committedNext, nativeTrimmedHashes));
|
|
7015
7275
|
const rollbackCommitted = async (cause, committedCids) => {
|
|
7016
7276
|
durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
@@ -7678,14 +7938,15 @@ let SharedLog = (() => {
|
|
|
7678
7938
|
const nativeTrimmedHashSet = new Set(committedAppends.flatMap((append) => append.trimmedHashes ??
|
|
7679
7939
|
append.trimmed.map((entry) => entry.hash)));
|
|
7680
7940
|
const nativeTrimmedHashes = [...nativeTrimmedHashSet];
|
|
7681
|
-
batchCoordinateRollback =
|
|
7682
|
-
|
|
7683
|
-
|
|
7684
|
-
|
|
7685
|
-
|
|
7686
|
-
|
|
7687
|
-
append.
|
|
7688
|
-
|
|
7941
|
+
batchCoordinateRollback =
|
|
7942
|
+
this._coordinates.snapshotResidentCoordinateEntries(committedAppends.flatMap((append) => [
|
|
7943
|
+
...((append.entry.cid ?? append.entry.hash)
|
|
7944
|
+
? [append.entry.cid ?? append.entry.hash]
|
|
7945
|
+
: []),
|
|
7946
|
+
...append.entry.next,
|
|
7947
|
+
...(append.trimmedHashes ??
|
|
7948
|
+
append.trimmed.map((entry) => entry.hash)),
|
|
7949
|
+
]));
|
|
7689
7950
|
await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, committedCids, nativeTrimmedHashes, batchCoordinateRollback, committedAppends.flatMap((append) => [
|
|
7690
7951
|
...append.entry.next,
|
|
7691
7952
|
...(append.trimmedHashes ??
|
|
@@ -8895,6 +9156,8 @@ let SharedLog = (() => {
|
|
|
8895
9156
|
ttl: LEADER_PLAN_CACHE_TTL_MS,
|
|
8896
9157
|
});
|
|
8897
9158
|
this._peerSyncCapabilities = new Map();
|
|
9159
|
+
this._peerSyncCapabilitySessions = new Map();
|
|
9160
|
+
this._peerSyncCapabilityTimestamps = new Map();
|
|
8898
9161
|
this._liveRawGossipBatches = new Map();
|
|
8899
9162
|
this._liveRawGossipFlushScheduled = false;
|
|
8900
9163
|
this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
|
|
@@ -8910,6 +9173,10 @@ let SharedLog = (() => {
|
|
|
8910
9173
|
this._lastLocalAppendAt = 0;
|
|
8911
9174
|
this._announcements ??= this.createReplicationAnnouncementCoordinator();
|
|
8912
9175
|
this._announcements.resetForOpen();
|
|
9176
|
+
this._v2Receive ??= this.createReplicationInfoV2ReceiveCoordinator();
|
|
9177
|
+
this._v2Receive.resetForOpen();
|
|
9178
|
+
this._v2Send ??= this.createReplicationInfoV2SendCoordinator();
|
|
9179
|
+
this._v2Send.resetForOpen();
|
|
8913
9180
|
const adaptiveReplicateOptions = options?.replicate && isAdaptiveReplicatorOption(options.replicate)
|
|
8914
9181
|
? options.replicate
|
|
8915
9182
|
: undefined;
|
|
@@ -9479,7 +9746,8 @@ let SharedLog = (() => {
|
|
|
9479
9746
|
// any stale coordinate or document state to this backbone.
|
|
9480
9747
|
await this._coordinates._nativeBackboneCoordinatePersistence.resumeDrop?.();
|
|
9481
9748
|
await this._coordinates._nativeBackboneCoordinatePersistence.hydrate(backbone);
|
|
9482
|
-
this._coordinates._nativeBackboneCoordinateJournalLastFlushMs =
|
|
9749
|
+
this._coordinates._nativeBackboneCoordinateJournalLastFlushMs =
|
|
9750
|
+
Date.now();
|
|
9483
9751
|
this.hydrateNativeCoordinateStateFromBackbone(backbone);
|
|
9484
9752
|
return;
|
|
9485
9753
|
}
|
|
@@ -9691,18 +9959,20 @@ let SharedLog = (() => {
|
|
|
9691
9959
|
}
|
|
9692
9960
|
if (this.node.directory != null &&
|
|
9693
9961
|
this._coordinates._nativeBackboneCoordinatePersistence) {
|
|
9694
|
-
if (this._coordinates._nativeBackboneCoordinatePersistence
|
|
9962
|
+
if (this._coordinates._nativeBackboneCoordinatePersistence
|
|
9963
|
+
.durableBarrier !== true ||
|
|
9695
9964
|
typeof this._nativeBackboneCoordinatePersistenceStore
|
|
9696
9965
|
?.durableBarrier !== "function") {
|
|
9697
9966
|
throw new Error("Durable nativeBackbone coordinate persistence requires an explicit physical durability barrier");
|
|
9698
9967
|
}
|
|
9699
9968
|
}
|
|
9700
9969
|
if (this._coordinates._nativeBackboneCoordinatePersistence &&
|
|
9701
|
-
(this._coordinates._nativeBackboneCoordinatePersistence
|
|
9702
|
-
null ||
|
|
9703
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
9704
|
-
null) &&
|
|
9705
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
9970
|
+
(this._coordinates._nativeBackboneCoordinatePersistence
|
|
9971
|
+
.compactMaxJournalBytes != null ||
|
|
9972
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
9973
|
+
.compactMaxJournalRecords != null) &&
|
|
9974
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
9975
|
+
.crashSafeCompaction !== true) {
|
|
9706
9976
|
// Durable custom adapters must explicitly advertise an atomic generation
|
|
9707
9977
|
// protocol before SharedLog permits automatic WAL compaction.
|
|
9708
9978
|
throw new Error("Durable native coordinate persistence compaction thresholds require crashSafeCompaction");
|
|
@@ -10025,10 +10295,18 @@ let SharedLog = (() => {
|
|
|
10025
10295
|
}
|
|
10026
10296
|
}
|
|
10027
10297
|
cleanupPeerDisconnectTracking(peerHash, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
|
|
10298
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
10299
|
+
const preserveV2Session = peerSession?.phase === "open" && peerSession.isActive();
|
|
10028
10300
|
this.cancelReplicationInfoRequests(peerHash);
|
|
10029
10301
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
10030
10302
|
this._liveness._replicatorLastActivityAt.delete(peerHash);
|
|
10031
|
-
|
|
10303
|
+
if (!preserveV2Session) {
|
|
10304
|
+
this._peerSyncCapabilities.delete(peerHash);
|
|
10305
|
+
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
10306
|
+
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
10307
|
+
this._v2Receive.clearPeer(peerHash);
|
|
10308
|
+
this._v2Send.clearPeer(peerHash);
|
|
10309
|
+
}
|
|
10032
10310
|
this.cleanupPendingIHavePeer(peerHash);
|
|
10033
10311
|
this.cleanupCheckedPrunePeer(peerHash, ownershipLifecycleController, this._checkedPrune);
|
|
10034
10312
|
}
|
|
@@ -10046,8 +10324,16 @@ let SharedLog = (() => {
|
|
|
10046
10324
|
// when they eventually reach the apply lane. Reset the sender's
|
|
10047
10325
|
// ordering watermark with the local epoch so a later arrival can be
|
|
10048
10326
|
// accepted without comparing its clock to this receiver's clock.
|
|
10049
|
-
this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
10327
|
+
const receiveEpoch = this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
10050
10328
|
this.latestReplicationInfoMessage.delete(peerHash);
|
|
10329
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
10330
|
+
if (peerSession?.phase === "open") {
|
|
10331
|
+
this._v2Receive.advanceRecovery({
|
|
10332
|
+
peerHash,
|
|
10333
|
+
peerSession,
|
|
10334
|
+
receiveEpoch,
|
|
10335
|
+
});
|
|
10336
|
+
}
|
|
10051
10337
|
}
|
|
10052
10338
|
async resolveCandidatePeersForHash(hash, options) {
|
|
10053
10339
|
if (options?.signal?.aborted)
|
|
@@ -10845,6 +11131,10 @@ let SharedLog = (() => {
|
|
|
10845
11131
|
this._openingSyncCapabilitiesByPeer?.clear();
|
|
10846
11132
|
this._gidPeersHistory?.clear();
|
|
10847
11133
|
this._peerSyncCapabilities?.clear();
|
|
11134
|
+
this._peerSyncCapabilitySessions?.clear();
|
|
11135
|
+
this._peerSyncCapabilityTimestamps?.clear();
|
|
11136
|
+
this._v2Receive?.clearForClose();
|
|
11137
|
+
this._v2Send?.clearForClose();
|
|
10848
11138
|
this._liveRawGossipBatches?.clear();
|
|
10849
11139
|
this._nativeSharedLogState?.clearGidPeers();
|
|
10850
11140
|
this._nativeBackbone?.clearGidPeers();
|
|
@@ -10907,6 +11197,7 @@ let SharedLog = (() => {
|
|
|
10907
11197
|
this.preventParentAttachments();
|
|
10908
11198
|
this.stopRepairLifecycle();
|
|
10909
11199
|
this._instanceLifecycle?.beginTerminal("close");
|
|
11200
|
+
this._v2Receive?.clearForClose();
|
|
10910
11201
|
const replicationRangeTerminalFence = this.acquireReplicationRangeMutationTerminalFence();
|
|
10911
11202
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
10912
11203
|
try {
|
|
@@ -10965,12 +11256,16 @@ let SharedLog = (() => {
|
|
|
10965
11256
|
}
|
|
10966
11257
|
}, 2_000);
|
|
10967
11258
|
try {
|
|
10968
|
-
|
|
10969
|
-
|
|
10970
|
-
|
|
10971
|
-
|
|
10972
|
-
|
|
10973
|
-
|
|
11259
|
+
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
11260
|
+
await Promise.all([
|
|
11261
|
+
this.rpc
|
|
11262
|
+
.send(reset, {
|
|
11263
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
11264
|
+
signal: abort.signal,
|
|
11265
|
+
})
|
|
11266
|
+
.catch(() => { }),
|
|
11267
|
+
this._v2Send.sendTerminalReset(abort.signal),
|
|
11268
|
+
]);
|
|
10974
11269
|
}
|
|
10975
11270
|
finally {
|
|
10976
11271
|
clearTimeout(abortTimer);
|
|
@@ -11047,6 +11342,7 @@ let SharedLog = (() => {
|
|
|
11047
11342
|
this.preventParentAttachments();
|
|
11048
11343
|
this.stopRepairLifecycle();
|
|
11049
11344
|
this._instanceLifecycle?.beginTerminal("drop");
|
|
11345
|
+
this._v2Receive?.clearForClose();
|
|
11050
11346
|
const replicationRangeTerminalFence = this.acquireReplicationRangeMutationTerminalFence();
|
|
11051
11347
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
11052
11348
|
try {
|
|
@@ -11088,12 +11384,16 @@ let SharedLog = (() => {
|
|
|
11088
11384
|
}
|
|
11089
11385
|
}, 2_000);
|
|
11090
11386
|
try {
|
|
11091
|
-
|
|
11092
|
-
|
|
11093
|
-
|
|
11094
|
-
|
|
11095
|
-
|
|
11096
|
-
|
|
11387
|
+
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
11388
|
+
await Promise.all([
|
|
11389
|
+
this.rpc
|
|
11390
|
+
.send(reset, {
|
|
11391
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
11392
|
+
signal: abort.signal,
|
|
11393
|
+
})
|
|
11394
|
+
.catch(() => { }),
|
|
11395
|
+
this._v2Send.sendTerminalReset(abort.signal),
|
|
11396
|
+
]);
|
|
11097
11397
|
}
|
|
11098
11398
|
finally {
|
|
11099
11399
|
clearTimeout(abortTimer);
|
|
@@ -11508,13 +11808,6 @@ let SharedLog = (() => {
|
|
|
11508
11808
|
}
|
|
11509
11809
|
// Callback for receiving a message from the network
|
|
11510
11810
|
async onMessage(msg, context) {
|
|
11511
|
-
// Decode-first only: capability advertisement is not permission to use
|
|
11512
|
-
// V2. Drop every unsolicited V2 envelope before even entering the shared
|
|
11513
|
-
// receive envelope, so durable poison checks, leases, liveness, ordering
|
|
11514
|
-
// watermarks, replication ranges and cleanup side effects cannot observe it.
|
|
11515
|
-
if (isReplicationInfoV2Message(msg)) {
|
|
11516
|
-
return;
|
|
11517
|
-
}
|
|
11518
11811
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
11519
11812
|
? msg
|
|
11520
11813
|
: undefined;
|
|
@@ -11555,17 +11848,25 @@ let SharedLog = (() => {
|
|
|
11555
11848
|
msg = msg.toReplicationInfoMessage(); // migration
|
|
11556
11849
|
}
|
|
11557
11850
|
if (msg instanceof AllReplicatingSegmentsMessage ||
|
|
11558
|
-
msg instanceof AddedReplicationSegmentMessage
|
|
11851
|
+
msg instanceof AddedReplicationSegmentMessage ||
|
|
11852
|
+
msg instanceof FullReplicationInfoV2Message ||
|
|
11853
|
+
msg instanceof AddedReplicationInfoV2Message) {
|
|
11559
11854
|
// Bound decoded untrusted vectors before per-peer/global mutation
|
|
11560
11855
|
// queues, trusted-replicator authorization, or liveness side effects.
|
|
11561
11856
|
this.validateReplicationRangeAnnouncement(msg.segments);
|
|
11562
11857
|
}
|
|
11563
|
-
else if (msg instanceof StoppedReplicating
|
|
11858
|
+
else if (msg instanceof StoppedReplicating ||
|
|
11859
|
+
msg instanceof StoppedReplicationInfoV2Message) {
|
|
11564
11860
|
// Bound the raw decoded vector before deduplication can hide the
|
|
11565
11861
|
// allocation cost, and before liveness or apply-queue side effects.
|
|
11566
11862
|
this.validateStoppedReplicationAnnouncement(msg.segmentIds);
|
|
11567
11863
|
}
|
|
11568
|
-
if (!context.from.equals(this.node.identity.publicKey)
|
|
11864
|
+
if (!context.from.equals(this.node.identity.publicKey) &&
|
|
11865
|
+
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
11866
|
+
!isReplicationInfoV2Message(msg) &&
|
|
11867
|
+
!(msg instanceof AllReplicatingSegmentsMessage) &&
|
|
11868
|
+
!(msg instanceof AddedReplicationSegmentMessage) &&
|
|
11869
|
+
!(msg instanceof StoppedReplicating)) {
|
|
11569
11870
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
11570
11871
|
}
|
|
11571
11872
|
const syncProfile = this._logProperties?.sync?.profile;
|
|
@@ -12881,6 +13182,7 @@ let SharedLog = (() => {
|
|
|
12881
13182
|
fromHash: receiveFromHash,
|
|
12882
13183
|
session: receiveSession,
|
|
12883
13184
|
lifecycleController: receiveReplicationLifecycleController,
|
|
13185
|
+
ownershipLifecycleController: receiveOwnershipLifecycleController,
|
|
12884
13186
|
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
12885
13187
|
syncProfile,
|
|
12886
13188
|
lease: peerReceiveLease,
|
|
@@ -12906,8 +13208,13 @@ let SharedLog = (() => {
|
|
|
12906
13208
|
this.clearRepairFrontierHashes(context.from.hashcode(), msg.hashes);
|
|
12907
13209
|
return;
|
|
12908
13210
|
}
|
|
13211
|
+
else if (isReplicationInfoV2Message(msg)) {
|
|
13212
|
+
await this.handleReplicationInfoV2Announcement(msg, laneRequestContext, lane);
|
|
13213
|
+
}
|
|
12909
13214
|
else if (msg instanceof SyncCapabilitiesMessage) {
|
|
12910
13215
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
13216
|
+
const capabilityTransportSession = context.message?.header?.session;
|
|
13217
|
+
const capabilityTimestamp = context.message?.header?.timestamp;
|
|
12911
13218
|
// No await separates this from the capture above, so the captured
|
|
12912
13219
|
// window state is exact: the legacy re-read of the opening map here
|
|
12913
13220
|
// could never observe a different value.
|
|
@@ -12916,17 +13223,49 @@ let SharedLog = (() => {
|
|
|
12916
13223
|
// A prior unsubscribe cleanup may still be ahead of this reconnect
|
|
12917
13224
|
// barrier. Stage the new generation's one-shot advertisement so that
|
|
12918
13225
|
// cleanup cannot erase it before the opening transition commits.
|
|
12919
|
-
this.
|
|
12920
|
-
|
|
13226
|
+
this.observePeerSyncCapabilities({
|
|
13227
|
+
peerHash: receiveFromHash,
|
|
12921
13228
|
capabilities: msg.capabilities,
|
|
13229
|
+
transportSession: capabilityTransportSession,
|
|
13230
|
+
timestamp: capabilityTimestamp,
|
|
13231
|
+
openingSession: receiveSession,
|
|
12922
13232
|
});
|
|
12923
13233
|
}
|
|
12924
|
-
else {
|
|
12925
|
-
|
|
13234
|
+
else if (this.observePeerSyncCapabilities({
|
|
13235
|
+
peerHash: receiveFromHash,
|
|
13236
|
+
capabilities: msg.capabilities,
|
|
13237
|
+
transportSession: capabilityTransportSession,
|
|
13238
|
+
timestamp: capabilityTimestamp,
|
|
13239
|
+
}) &&
|
|
13240
|
+
receiveSession?.phase === "open") {
|
|
13241
|
+
this.promoteReplicationInfoV2ReceiveCapability(context.from, receiveSession);
|
|
12926
13242
|
}
|
|
12927
13243
|
}
|
|
12928
13244
|
return;
|
|
12929
13245
|
}
|
|
13246
|
+
else if (msg instanceof RequestReplicationInfoV2Message) {
|
|
13247
|
+
const requiredCapabilities = SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
13248
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY;
|
|
13249
|
+
if (receiveSession &&
|
|
13250
|
+
((this._peerSyncCapabilities.get(receiveFromHash) ?? 0) &
|
|
13251
|
+
requiredCapabilities) ===
|
|
13252
|
+
requiredCapabilities &&
|
|
13253
|
+
this._peerSyncCapabilitySessions.get(receiveFromHash) ===
|
|
13254
|
+
context.message.header.session &&
|
|
13255
|
+
this._peerSyncCapabilityTimestamps.has(receiveFromHash) &&
|
|
13256
|
+
context.message.header.timestamp >
|
|
13257
|
+
this._peerSyncCapabilityTimestamps.get(receiveFromHash) &&
|
|
13258
|
+
this._v2Send.acceptRequest(msg, {
|
|
13259
|
+
from: context.from,
|
|
13260
|
+
peerSession: receiveSession,
|
|
13261
|
+
receiverTransportSession: context.message.header.session,
|
|
13262
|
+
capabilityTimestamp: this._peerSyncCapabilityTimestamps.get(receiveFromHash),
|
|
13263
|
+
requestTimestamp: context.message.header.timestamp,
|
|
13264
|
+
})) {
|
|
13265
|
+
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
13266
|
+
}
|
|
13267
|
+
return;
|
|
13268
|
+
}
|
|
12930
13269
|
else if (await this.syncronizer.onMessage(msg, context)) {
|
|
12931
13270
|
return; // the syncronizer has handled the message
|
|
12932
13271
|
}
|
|
@@ -13162,6 +13501,7 @@ let SharedLog = (() => {
|
|
|
13162
13501
|
}
|
|
13163
13502
|
const segments = replicationSegments.map((x) => x.toReplicationRange());
|
|
13164
13503
|
this.validatePersistedReplicationRangeSnapshot(segments);
|
|
13504
|
+
this._v2Send.enqueueSnapshotForPeer(receiveFromHash);
|
|
13165
13505
|
await this.rpc
|
|
13166
13506
|
.send(new AllReplicatingSegmentsMessage({ segments }), {
|
|
13167
13507
|
mode: new AcknowledgeDelivery({
|
|
@@ -13195,6 +13535,147 @@ let SharedLog = (() => {
|
|
|
13195
13535
|
}
|
|
13196
13536
|
}
|
|
13197
13537
|
}
|
|
13538
|
+
async handleReplicationInfoV2Announcement(msg, context, lane) {
|
|
13539
|
+
const from = context.from;
|
|
13540
|
+
const fromHash = lane.fromHash;
|
|
13541
|
+
const receiveSession = lane.session;
|
|
13542
|
+
if (from.equals(this.node.identity.publicKey) ||
|
|
13543
|
+
receiveSession === null ||
|
|
13544
|
+
receiveSession.phase !== "open") {
|
|
13545
|
+
return;
|
|
13546
|
+
}
|
|
13547
|
+
// Authenticate and reserve before the per-peer lane. One reservation per
|
|
13548
|
+
// peer bounds decoded-frame retention while another mutation is parked.
|
|
13549
|
+
const receiveState = this._v2Receive._receiveStates.get(fromHash);
|
|
13550
|
+
if (!receiveState ||
|
|
13551
|
+
receiveState.peerSession !== receiveSession ||
|
|
13552
|
+
receiveState.senderTransportSession !== context.message.header.session) {
|
|
13553
|
+
return;
|
|
13554
|
+
}
|
|
13555
|
+
const admission = this._v2Receive.reserve(msg, {
|
|
13556
|
+
from,
|
|
13557
|
+
peerSession: receiveSession,
|
|
13558
|
+
receiveEpoch: lane.receiveEpoch,
|
|
13559
|
+
senderTransportSession: context.message.header.session,
|
|
13560
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13561
|
+
});
|
|
13562
|
+
if (!admission) {
|
|
13563
|
+
return;
|
|
13564
|
+
}
|
|
13565
|
+
lane.lease.release();
|
|
13566
|
+
try {
|
|
13567
|
+
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
13568
|
+
const hostGate = () => this._instanceLifecycle.isMembershipActiveFor(lane.lifecycleController) &&
|
|
13569
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController) &&
|
|
13570
|
+
this._peerSessions.isCurrent(fromHash, receiveSession) &&
|
|
13571
|
+
receiveSession.phase === "open" &&
|
|
13572
|
+
this._peerSessions.isReceiveEpochCurrent(fromHash, lane.receiveEpoch) &&
|
|
13573
|
+
!this._peerSessions.isReplicationInfoBlocked(fromHash) &&
|
|
13574
|
+
this._peerSessions.isReceiveCleanupGateOpen(fromHash);
|
|
13575
|
+
if (!hostGate()) {
|
|
13576
|
+
return;
|
|
13577
|
+
}
|
|
13578
|
+
const exactGate = () => hostGate() && this._v2Receive.isAdmissionCurrent(admission);
|
|
13579
|
+
let durableCommitted = false;
|
|
13580
|
+
try {
|
|
13581
|
+
if (msg instanceof FullReplicationInfoV2Message ||
|
|
13582
|
+
msg instanceof AddedReplicationInfoV2Message) {
|
|
13583
|
+
let mutationGateChecked = false;
|
|
13584
|
+
let mutationGateAdmitted = false;
|
|
13585
|
+
const result = await this.addReplicationRange(msg.segments.map((segment) => segment.toReplicationRangeIndexable(from)), from, {
|
|
13586
|
+
reset: msg instanceof FullReplicationInfoV2Message,
|
|
13587
|
+
checkDuplicates: true,
|
|
13588
|
+
timestamp: Number(context.message.header.timestamp),
|
|
13589
|
+
allowLegacyOrderedReplacementPairs: msg instanceof AddedReplicationInfoV2Message,
|
|
13590
|
+
shouldApply: () => {
|
|
13591
|
+
mutationGateChecked = true;
|
|
13592
|
+
mutationGateAdmitted = exactGate();
|
|
13593
|
+
return mutationGateAdmitted;
|
|
13594
|
+
},
|
|
13595
|
+
onDurableApplyCommitted: () => {
|
|
13596
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
13597
|
+
return false;
|
|
13598
|
+
}
|
|
13599
|
+
durableCommitted = true;
|
|
13600
|
+
return true;
|
|
13601
|
+
},
|
|
13602
|
+
}, lane.ownershipLifecycleController);
|
|
13603
|
+
if (result === undefined ||
|
|
13604
|
+
!mutationGateChecked ||
|
|
13605
|
+
!mutationGateAdmitted ||
|
|
13606
|
+
!durableCommitted ||
|
|
13607
|
+
!exactGate()) {
|
|
13608
|
+
return;
|
|
13609
|
+
}
|
|
13610
|
+
}
|
|
13611
|
+
else {
|
|
13612
|
+
if (!exactGate()) {
|
|
13613
|
+
return;
|
|
13614
|
+
}
|
|
13615
|
+
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(msg.segmentIds, from);
|
|
13616
|
+
if (!exactGate()) {
|
|
13617
|
+
return;
|
|
13618
|
+
}
|
|
13619
|
+
let mutationGateAdmitted = true;
|
|
13620
|
+
const removedRanges = [];
|
|
13621
|
+
const removed = await this.removeReplicationRanges(rangesToRemove, from, {
|
|
13622
|
+
shouldRemove: () => {
|
|
13623
|
+
mutationGateAdmitted = exactGate();
|
|
13624
|
+
return mutationGateAdmitted;
|
|
13625
|
+
},
|
|
13626
|
+
onDurableRemoveCommitted: () => {
|
|
13627
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
13628
|
+
return false;
|
|
13629
|
+
}
|
|
13630
|
+
durableCommitted = true;
|
|
13631
|
+
return true;
|
|
13632
|
+
},
|
|
13633
|
+
onRemoved: (ranges) => removedRanges.push(...ranges),
|
|
13634
|
+
}, lane.ownershipLifecycleController);
|
|
13635
|
+
if (!durableCommitted) {
|
|
13636
|
+
if (!mutationGateAdmitted ||
|
|
13637
|
+
!exactGate() ||
|
|
13638
|
+
removed ||
|
|
13639
|
+
!this._v2Receive.commit(admission)) {
|
|
13640
|
+
return;
|
|
13641
|
+
}
|
|
13642
|
+
durableCommitted = true;
|
|
13643
|
+
}
|
|
13644
|
+
if (this._instanceLifecycle.isMembershipActiveFor(lane.lifecycleController) &&
|
|
13645
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController)) {
|
|
13646
|
+
const timestamp = BigInt(Date.now());
|
|
13647
|
+
for (const range of removedRanges) {
|
|
13648
|
+
this.replicationChangeDebounceFn.add({
|
|
13649
|
+
range,
|
|
13650
|
+
type: "removed",
|
|
13651
|
+
timestamp,
|
|
13652
|
+
});
|
|
13653
|
+
}
|
|
13654
|
+
}
|
|
13655
|
+
}
|
|
13656
|
+
}
|
|
13657
|
+
catch (error) {
|
|
13658
|
+
this._v2Receive.requireFullAfterFailure(admission);
|
|
13659
|
+
throw error;
|
|
13660
|
+
}
|
|
13661
|
+
if (!durableCommitted) {
|
|
13662
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
13663
|
+
return;
|
|
13664
|
+
}
|
|
13665
|
+
}
|
|
13666
|
+
if (!hostGate()) {
|
|
13667
|
+
return;
|
|
13668
|
+
}
|
|
13669
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
13670
|
+
if (msg instanceof FullReplicationInfoV2Message) {
|
|
13671
|
+
this.cancelReplicationInfoRequests(fromHash);
|
|
13672
|
+
}
|
|
13673
|
+
});
|
|
13674
|
+
}
|
|
13675
|
+
finally {
|
|
13676
|
+
this._v2Receive.release(admission);
|
|
13677
|
+
}
|
|
13678
|
+
}
|
|
13198
13679
|
async handleReplicationInfoAnnouncement(msg, context, lane) {
|
|
13199
13680
|
const receiveFromHash = lane.fromHash;
|
|
13200
13681
|
const receiveSession = lane.session;
|
|
@@ -13210,6 +13691,19 @@ let SharedLog = (() => {
|
|
|
13210
13691
|
// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
|
|
13211
13692
|
const from = context.from;
|
|
13212
13693
|
const fromHash = from.hashcode();
|
|
13694
|
+
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
13695
|
+
if (receiveSession) {
|
|
13696
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
13697
|
+
peerHash: fromHash,
|
|
13698
|
+
peerSession: receiveSession,
|
|
13699
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
13700
|
+
senderTransportSession: context.message.header.session,
|
|
13701
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13702
|
+
message: msg,
|
|
13703
|
+
});
|
|
13704
|
+
}
|
|
13705
|
+
return;
|
|
13706
|
+
}
|
|
13213
13707
|
// Pre-lane gate: lifecycle -> receive-epoch -> blocked, exactly the
|
|
13214
13708
|
// legacy order. isMembershipActiveFor is the unit-pinned fold of
|
|
13215
13709
|
// isReplicationLifecycleActive; isReceiveEpochCurrent is the
|
|
@@ -13234,6 +13728,17 @@ let SharedLog = (() => {
|
|
|
13234
13728
|
this._peerSessions.isReplicationInfoBlocked(fromHash)) {
|
|
13235
13729
|
return;
|
|
13236
13730
|
}
|
|
13731
|
+
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
13732
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
13733
|
+
peerHash: fromHash,
|
|
13734
|
+
peerSession: receiveSession,
|
|
13735
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
13736
|
+
senderTransportSession: context.message.header.session,
|
|
13737
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13738
|
+
message: msg,
|
|
13739
|
+
});
|
|
13740
|
+
return;
|
|
13741
|
+
}
|
|
13237
13742
|
// Process in-order to avoid races where repeated reset messages arrive
|
|
13238
13743
|
// concurrently and trigger spurious "added" diffs / rebalancing.
|
|
13239
13744
|
const prev = this.latestReplicationInfoMessage.get(fromHash);
|
|
@@ -13241,17 +13746,20 @@ let SharedLog = (() => {
|
|
|
13241
13746
|
return;
|
|
13242
13747
|
}
|
|
13243
13748
|
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
13244
|
-
this._liveness._replicatorLivenessFailures.delete(fromHash);
|
|
13245
13749
|
if (this.closed) {
|
|
13246
13750
|
return;
|
|
13247
13751
|
}
|
|
13248
13752
|
const reset = msg instanceof AllReplicatingSegmentsMessage;
|
|
13249
|
-
await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(from)), from, {
|
|
13753
|
+
const result = await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(from)), from, {
|
|
13250
13754
|
reset,
|
|
13251
13755
|
checkDuplicates: true,
|
|
13252
13756
|
timestamp: Number(messageTimestamp),
|
|
13253
13757
|
allowLegacyOrderedReplacementPairs: msg instanceof AddedReplicationSegmentMessage,
|
|
13254
13758
|
});
|
|
13759
|
+
if (result === undefined) {
|
|
13760
|
+
return;
|
|
13761
|
+
}
|
|
13762
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
13255
13763
|
// If the peer reports any replication segments, stop re-requesting.
|
|
13256
13764
|
// (Empty reports can be transient during startup.)
|
|
13257
13765
|
if (replicationInfoMessage.segments.length > 0) {
|
|
@@ -13278,6 +13786,19 @@ let SharedLog = (() => {
|
|
|
13278
13786
|
return;
|
|
13279
13787
|
}
|
|
13280
13788
|
const fromHash = from.hashcode();
|
|
13789
|
+
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
13790
|
+
if (receiveSession) {
|
|
13791
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
13792
|
+
peerHash: fromHash,
|
|
13793
|
+
peerSession: receiveSession,
|
|
13794
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
13795
|
+
senderTransportSession: context.message.header.session,
|
|
13796
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13797
|
+
message: msg,
|
|
13798
|
+
});
|
|
13799
|
+
}
|
|
13800
|
+
return;
|
|
13801
|
+
}
|
|
13281
13802
|
// Same pre-lane gate shape as Added/All above (and the same
|
|
13282
13803
|
// intentional absence of a subscription-epoch term).
|
|
13283
13804
|
if (!this._instanceLifecycle.isMembershipActiveFor(receiveReplicationLifecycleController) ||
|
|
@@ -13294,17 +13815,28 @@ let SharedLog = (() => {
|
|
|
13294
13815
|
this._peerSessions.isReplicationInfoBlocked(fromHash)) {
|
|
13295
13816
|
return;
|
|
13296
13817
|
}
|
|
13818
|
+
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
13819
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
13820
|
+
peerHash: fromHash,
|
|
13821
|
+
peerSession: receiveSession,
|
|
13822
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
13823
|
+
senderTransportSession: context.message.header.session,
|
|
13824
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13825
|
+
message: msg,
|
|
13826
|
+
});
|
|
13827
|
+
return;
|
|
13828
|
+
}
|
|
13297
13829
|
const previousTimestamp = this.latestReplicationInfoMessage.get(fromHash);
|
|
13298
13830
|
if (previousTimestamp && previousTimestamp > messageTimestamp) {
|
|
13299
13831
|
return;
|
|
13300
13832
|
}
|
|
13301
13833
|
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
13302
|
-
this._liveness._replicatorLivenessFailures.delete(fromHash);
|
|
13303
13834
|
if (this.closed) {
|
|
13304
13835
|
return;
|
|
13305
13836
|
}
|
|
13306
13837
|
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(segmentIds, from);
|
|
13307
13838
|
await this.removeReplicationRanges(rangesToRemove, from);
|
|
13839
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
13308
13840
|
const timestamp = BigInt(+new Date());
|
|
13309
13841
|
for (const range of rangesToRemove) {
|
|
13310
13842
|
this.replicationChangeDebounceFn.add({
|
|
@@ -15173,6 +15705,8 @@ let SharedLog = (() => {
|
|
|
15173
15705
|
* This method must run inside the global replication-range mutation lane.
|
|
15174
15706
|
*/
|
|
15175
15707
|
async deleteReplicationRangesCoherently(ranges, ownerHash, options) {
|
|
15708
|
+
const wasReplicator = this.uniqueReplicators.has(ownerHash);
|
|
15709
|
+
const joinWasEmitted = this._replicatorJoinEmitted.has(ownerHash);
|
|
15176
15710
|
if (ranges.length === 0) {
|
|
15177
15711
|
const ownerHasRanges = (await this.replicationIndex.count({ query: { hash: ownerHash } })) > 0;
|
|
15178
15712
|
if (!ownerHasRanges) {
|
|
@@ -15183,13 +15717,25 @@ let SharedLog = (() => {
|
|
|
15183
15717
|
removed: [],
|
|
15184
15718
|
retained: [],
|
|
15185
15719
|
ownerHasRanges,
|
|
15720
|
+
rollback: async () => {
|
|
15721
|
+
if (wasReplicator) {
|
|
15722
|
+
this.uniqueReplicators.add(ownerHash);
|
|
15723
|
+
}
|
|
15724
|
+
else {
|
|
15725
|
+
this.uniqueReplicators.delete(ownerHash);
|
|
15726
|
+
}
|
|
15727
|
+
if (joinWasEmitted) {
|
|
15728
|
+
this._replicatorJoinEmitted.add(ownerHash);
|
|
15729
|
+
}
|
|
15730
|
+
else {
|
|
15731
|
+
this._replicatorJoinEmitted.delete(ownerHash);
|
|
15732
|
+
}
|
|
15733
|
+
},
|
|
15186
15734
|
};
|
|
15187
15735
|
}
|
|
15188
15736
|
const uniqueRanges = [
|
|
15189
15737
|
...new Map(ranges.map((range) => [range.idString, range])).values(),
|
|
15190
15738
|
];
|
|
15191
|
-
const wasReplicator = this.uniqueReplicators.has(ownerHash);
|
|
15192
|
-
const joinWasEmitted = this._replicatorJoinEmitted.has(ownerHash);
|
|
15193
15739
|
const snapshots = uniqueRanges.map((range) => {
|
|
15194
15740
|
const pending = this.pendingMaturity.get(ownerHash)?.get(range.idString);
|
|
15195
15741
|
return {
|
|
@@ -15376,10 +15922,76 @@ let SharedLog = (() => {
|
|
|
15376
15922
|
outcomeError = new AggregateError(errors, "Replication-range deletion and post-state reconciliation failed");
|
|
15377
15923
|
this.poisonReplicationOwnership(outcomeError);
|
|
15378
15924
|
}
|
|
15925
|
+
let rollbackStarted = false;
|
|
15926
|
+
const rollback = async () => {
|
|
15927
|
+
if (rollbackStarted) {
|
|
15928
|
+
return;
|
|
15929
|
+
}
|
|
15930
|
+
rollbackStarted = true;
|
|
15931
|
+
const rollbackErrors = [];
|
|
15932
|
+
const removedIds = new Set(removed.map((range) => range.idString));
|
|
15933
|
+
for (const snapshot of snapshots) {
|
|
15934
|
+
if (!removedIds.has(snapshot.range.idString)) {
|
|
15935
|
+
continue;
|
|
15936
|
+
}
|
|
15937
|
+
try {
|
|
15938
|
+
await this.replicationIndex.put(snapshot.range);
|
|
15939
|
+
this.putNativeReplicationRange(snapshot.range);
|
|
15940
|
+
}
|
|
15941
|
+
catch (error) {
|
|
15942
|
+
rollbackErrors.push(error);
|
|
15943
|
+
}
|
|
15944
|
+
}
|
|
15945
|
+
for (const snapshot of snapshots) {
|
|
15946
|
+
if (!removedIds.has(snapshot.range.idString) ||
|
|
15947
|
+
!snapshot.pending ||
|
|
15948
|
+
this.pendingMaturity.get(ownerHash)?.has(snapshot.range.idString)) {
|
|
15949
|
+
continue;
|
|
15950
|
+
}
|
|
15951
|
+
try {
|
|
15952
|
+
this.schedulePendingMaturity(snapshot.pending.range, snapshot.pending.from, {
|
|
15953
|
+
rebalance: snapshot.pending.rebalance,
|
|
15954
|
+
waitMs: Math.max(0, snapshot.pending.expiresAt - Date.now()),
|
|
15955
|
+
}, snapshot.pending.ownershipLifecycleController);
|
|
15956
|
+
}
|
|
15957
|
+
catch (error) {
|
|
15958
|
+
rollbackErrors.push(error);
|
|
15959
|
+
}
|
|
15960
|
+
}
|
|
15961
|
+
try {
|
|
15962
|
+
if (wasReplicator) {
|
|
15963
|
+
this.uniqueReplicators.add(ownerHash);
|
|
15964
|
+
}
|
|
15965
|
+
else {
|
|
15966
|
+
this.uniqueReplicators.delete(ownerHash);
|
|
15967
|
+
}
|
|
15968
|
+
if (joinWasEmitted) {
|
|
15969
|
+
this._replicatorJoinEmitted.add(ownerHash);
|
|
15970
|
+
}
|
|
15971
|
+
else {
|
|
15972
|
+
this._replicatorJoinEmitted.delete(ownerHash);
|
|
15973
|
+
}
|
|
15974
|
+
}
|
|
15975
|
+
catch (error) {
|
|
15976
|
+
rollbackErrors.push(error);
|
|
15977
|
+
}
|
|
15978
|
+
try {
|
|
15979
|
+
await this.updateOldestTimestampFromIndex();
|
|
15980
|
+
}
|
|
15981
|
+
catch (error) {
|
|
15982
|
+
rollbackErrors.push(error);
|
|
15983
|
+
}
|
|
15984
|
+
if (rollbackErrors.length > 0) {
|
|
15985
|
+
const failure = new AggregateError(rollbackErrors, "Replication-range deletion rollback failed");
|
|
15986
|
+
this.poisonReplicationOwnership(failure);
|
|
15987
|
+
throw failure;
|
|
15988
|
+
}
|
|
15989
|
+
};
|
|
15379
15990
|
return {
|
|
15380
15991
|
removed,
|
|
15381
15992
|
retained,
|
|
15382
15993
|
ownerHasRanges,
|
|
15994
|
+
rollback,
|
|
15383
15995
|
error: outcomeError,
|
|
15384
15996
|
};
|
|
15385
15997
|
}
|
|
@@ -15392,7 +16004,8 @@ let SharedLog = (() => {
|
|
|
15392
16004
|
}
|
|
15393
16005
|
this._replicationInfoRequestByPeer.delete(peerHash);
|
|
15394
16006
|
}
|
|
15395
|
-
scheduleReplicationInfoRequests(peer, replicationLifecycleController = this._instanceLifecycle
|
|
16007
|
+
scheduleReplicationInfoRequests(peer, replicationLifecycleController = this._instanceLifecycle
|
|
16008
|
+
?.membershipLifecycleController) {
|
|
15396
16009
|
if (!replicationLifecycleController ||
|
|
15397
16010
|
!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
15398
16011
|
return;
|
|
@@ -15463,6 +16076,13 @@ let SharedLog = (() => {
|
|
|
15463
16076
|
if (!ownsSubscriptionEpoch()) {
|
|
15464
16077
|
return;
|
|
15465
16078
|
}
|
|
16079
|
+
// A destination stream is scoped to exactly one topic-subscription
|
|
16080
|
+
// session. Abort the predecessor synchronously before either barrier can
|
|
16081
|
+
// yield; a late queue completion must never enter the new session.
|
|
16082
|
+
this._v2Receive.clearPeer(peerHash);
|
|
16083
|
+
this._v2Send.clearPeer(peerHash);
|
|
16084
|
+
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
16085
|
+
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
15466
16086
|
if (subscribed) {
|
|
15467
16087
|
const pendingOpeningCapabilities = this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
15468
16088
|
if (pendingOpeningCapabilities &&
|
|
@@ -15493,6 +16113,18 @@ let SharedLog = (() => {
|
|
|
15493
16113
|
const openingCapabilities = this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
15494
16114
|
if (openingCapabilities?.epoch === expectedSubscriptionEpoch) {
|
|
15495
16115
|
this._peerSyncCapabilities.set(peerHash, openingCapabilities.capabilities);
|
|
16116
|
+
if (openingCapabilities.transportSession === undefined) {
|
|
16117
|
+
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
16118
|
+
}
|
|
16119
|
+
else {
|
|
16120
|
+
this._peerSyncCapabilitySessions.set(peerHash, openingCapabilities.transportSession);
|
|
16121
|
+
}
|
|
16122
|
+
if (openingCapabilities.timestamp === undefined) {
|
|
16123
|
+
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
16124
|
+
}
|
|
16125
|
+
else {
|
|
16126
|
+
this._peerSyncCapabilityTimestamps.set(peerHash, openingCapabilities.timestamp);
|
|
16127
|
+
}
|
|
15496
16128
|
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
15497
16129
|
}
|
|
15498
16130
|
this._peerSessions.unblockReplicationInfo(peerHash);
|
|
@@ -15562,21 +16194,22 @@ let SharedLog = (() => {
|
|
|
15562
16194
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
15563
16195
|
this._liveness.markReplicatorActivity(peerHash);
|
|
15564
16196
|
this._peerSessions.markOpen(peerHash, expectedSubscriptionEpoch);
|
|
15565
|
-
|
|
15566
|
-
//
|
|
15567
|
-
//
|
|
15568
|
-
|
|
15569
|
-
|
|
15570
|
-
|
|
15571
|
-
|
|
15572
|
-
this.
|
|
15573
|
-
|
|
15574
|
-
|
|
15575
|
-
|
|
15576
|
-
mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
|
|
16197
|
+
this.promoteReplicationInfoV2ReceiveCapability(publicKey, expectedSubscriptionEpoch);
|
|
16198
|
+
// Decode, sender and authenticated apply readiness are separate bits. Start
|
|
16199
|
+
// the ACKed receiver-led negotiation before the legacy snapshot, but do not
|
|
16200
|
+
// hold mixed-version replication discovery behind that round trip. Attach
|
|
16201
|
+
// the rejection handler now because a lifecycle fence below may return before
|
|
16202
|
+
// awaiting the advert. RequestV2 is armed only after both sends settle.
|
|
16203
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
16204
|
+
const localCapabilityAdvertisement = this.advertiseReplicationInfoV2ReceiveCapability({
|
|
16205
|
+
target: publicKey,
|
|
16206
|
+
peerSession: expectedSubscriptionEpoch,
|
|
16207
|
+
receiveEpoch,
|
|
15577
16208
|
signal: replicationLifecycleController.signal,
|
|
15578
|
-
})
|
|
15579
|
-
|
|
16209
|
+
}).catch((error) => {
|
|
16210
|
+
this.handleReplicationLifecycleSendError(error, replicationLifecycleController);
|
|
16211
|
+
return undefined;
|
|
16212
|
+
});
|
|
15580
16213
|
let replicationSegments;
|
|
15581
16214
|
try {
|
|
15582
16215
|
replicationSegments = await this.getMyReplicationSegments();
|
|
@@ -15622,13 +16255,22 @@ let SharedLog = (() => {
|
|
|
15622
16255
|
.catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
|
|
15623
16256
|
}
|
|
15624
16257
|
}
|
|
15625
|
-
//
|
|
15626
|
-
// timing-sensitive delivery/order
|
|
15627
|
-
//
|
|
16258
|
+
// Keep legacy request-based discovery independent of the capability ACK too.
|
|
16259
|
+
// This makes mixed-version joins resilient to timing-sensitive delivery/order
|
|
16260
|
+
// issues where we may miss the remote peer's initial announcement.
|
|
15628
16261
|
if (this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
15629
16262
|
ownsSubscriptionEpoch()) {
|
|
15630
16263
|
this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
|
|
15631
16264
|
}
|
|
16265
|
+
const localCapabilityReady = await localCapabilityAdvertisement;
|
|
16266
|
+
if (localCapabilityReady) {
|
|
16267
|
+
this._v2Receive.markLocalCapabilityReady({
|
|
16268
|
+
peerHash,
|
|
16269
|
+
peerSession: expectedSubscriptionEpoch,
|
|
16270
|
+
receiveEpoch,
|
|
16271
|
+
...localCapabilityReady,
|
|
16272
|
+
});
|
|
16273
|
+
}
|
|
15632
16274
|
}
|
|
15633
16275
|
getClampedReplicas(customValue) {
|
|
15634
16276
|
if (!customValue) {
|