@peerbit/shared-log 13.2.32 → 13.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +6 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +613 -123
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-info-v2-binding.d.ts +16 -0
- package/dist/src/replication-info-v2-binding.d.ts.map +1 -0
- package/dist/src/replication-info-v2-binding.js +30 -0
- package/dist/src/replication-info-v2-binding.js.map +1 -0
- package/dist/src/replication-info-v2-receive.d.ts +250 -0
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -0
- package/dist/src/replication-info-v2-receive.js +1089 -0
- package/dist/src/replication-info-v2-receive.js.map +1 -0
- package/dist/src/replication-info-v2-send.d.ts +25 -17
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +170 -86
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/package.json +8 -8
- package/src/index.ts +897 -260
- package/src/replication-info-v2-binding.ts +44 -0
- package/src/replication-info-v2-receive.ts +1577 -0
- package/src/replication-info-v2-send.ts +223 -109
package/dist/src/index.js
CHANGED
|
@@ -51,27 +51,28 @@ import PQueue from "p-queue";
|
|
|
51
51
|
import { concat, fromString } from "uint8arrays";
|
|
52
52
|
import { BlocksMessage } from "./blocks.js";
|
|
53
53
|
import { CheckedPruneCoordinator, } from "./checked-prune.js";
|
|
54
|
+
import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isPromiseLike, mapMaybePromise, normalizedHashValues, } from "./coordinate-persistence.js";
|
|
54
55
|
import { CPUUsageIntervalLag } from "./cpu.js";
|
|
55
56
|
import { debouncedAccumulatorMap, } from "./debounce.js";
|
|
56
57
|
import { NativeDurableCommitError, NoPeersError, isNotStartedError, } from "./errors.js";
|
|
57
58
|
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
58
59
|
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
59
|
-
import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isPromiseLike, mapMaybePromise, normalizedHashValues, } from "./coordinate-persistence.js";
|
|
60
60
|
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
61
61
|
import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
|
|
62
62
|
import { JoinWarmupCoordinator } from "./join-warmup.js";
|
|
63
63
|
import { LeaderPlanCache } from "./leader-plan-cache.js";
|
|
64
64
|
import { TransportMessage } from "./message.js";
|
|
65
65
|
import { NativeBackboneWriteThroughBlockStore } from "./native-write-through-block-store.js";
|
|
66
|
-
import { PeerSessionRegistry
|
|
66
|
+
import { PeerSessionRegistry } from "./peer-session.js";
|
|
67
67
|
import { PIDReplicationController } from "./pid.js";
|
|
68
68
|
import { EntryReplicatedU32, EntryReplicatedU64, ReplicationIntent, ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, ReplicationRangeMessage, appromixateCoverage, calculateCoverage, countCoveringRangesSameOwner, createAssignedRangesQuery, debounceAggregationChanges, getAllMergeCandiates, getCoverSet, getSamples, isEntryReplicated, isMatured, isReplicationRangeMessage, mergeRanges, minimumWidthToCover, shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary, toRebalance, } from "./ranges.js";
|
|
69
69
|
import { ReplicationAnnouncementCoordinator, isTransientReplicationAnnouncementError, } from "./replication-announcement.js";
|
|
70
|
-
import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
|
|
71
70
|
import { createReplicationDomainHash, } from "./replication-domain-hash.js";
|
|
72
71
|
import { createReplicationDomainTime, } from "./replication-domain-time.js";
|
|
73
72
|
import {} from "./replication-domain.js";
|
|
74
|
-
import {
|
|
73
|
+
import { ReplicationInfoV2ReceiveCoordinator } from "./replication-info-v2-receive.js";
|
|
74
|
+
import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
|
|
75
|
+
import { AbsoluteReplicas, AddedReplicationInfoV2Message, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, FullReplicationInfoV2Message, MinReplicas, ReplicationError, ReplicationPingMessage, RequestReplicationInfoMessage, RequestReplicationInfoV2Message, ResponseRoleMessage, StoppedReplicating, StoppedReplicationInfoV2Message, decodeReplicas, encodeReplicas, isReplicationInfoV2Message, maxReplicas, } from "./replication.js";
|
|
75
76
|
import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
|
|
76
77
|
import { Observer, Replicator } from "./role.js";
|
|
77
78
|
import { createSyncronizer } from "./sync/factory.js";
|
|
@@ -625,7 +626,8 @@ let SharedLog = (() => {
|
|
|
625
626
|
value;
|
|
626
627
|
}
|
|
627
628
|
get _nativeBackboneCoordinateJournalLastFlushMs() {
|
|
628
|
-
return this._coordinates
|
|
629
|
+
return this._coordinates
|
|
630
|
+
?._nativeBackboneCoordinateJournalLastFlushMs;
|
|
629
631
|
}
|
|
630
632
|
set _nativeBackboneCoordinateJournalLastFlushMs(value) {
|
|
631
633
|
(this._coordinates ??=
|
|
@@ -1498,7 +1500,8 @@ let SharedLog = (() => {
|
|
|
1498
1500
|
}
|
|
1499
1501
|
if (!lowerMarkerCommitted) {
|
|
1500
1502
|
if (intent.coordinates.length > 0) {
|
|
1501
|
-
const mutationGenerations = (this._coordinates._nativeCoordinateMutationGenerations ??=
|
|
1503
|
+
const mutationGenerations = (this._coordinates._nativeCoordinateMutationGenerations ??=
|
|
1504
|
+
new Map());
|
|
1502
1505
|
const rollback = {
|
|
1503
1506
|
hashes: new Set(),
|
|
1504
1507
|
entries: new Map(),
|
|
@@ -1609,6 +1612,7 @@ let SharedLog = (() => {
|
|
|
1609
1612
|
// A fn that we can call many times that recalculates the participation role
|
|
1610
1613
|
rebalanceParticipationDebounced;
|
|
1611
1614
|
_announcements;
|
|
1615
|
+
_v2Receive;
|
|
1612
1616
|
_v2Send;
|
|
1613
1617
|
// A fn for debouncing the calls for pruning
|
|
1614
1618
|
pruneDebouncedFn;
|
|
@@ -1719,6 +1723,7 @@ let SharedLog = (() => {
|
|
|
1719
1723
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
1720
1724
|
validatePersistedReplicationRangeSnapshot: (ranges) => this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
1721
1725
|
isClosed: () => this.closed,
|
|
1726
|
+
isPeerSessionCurrent: (peerHash, peerSession) => this._peerSessions.isCurrent(peerHash, peerSession),
|
|
1722
1727
|
isPeerSessionOpen: (peerHash, peerSession) => this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
1723
1728
|
peerSession.phase === "open" &&
|
|
1724
1729
|
!this._peerSessions.isReplicationInfoBlocked(peerHash) &&
|
|
@@ -1727,6 +1732,83 @@ let SharedLog = (() => {
|
|
|
1727
1732
|
isReplicationOwnershipLifecycleActive: (controller) => this.isRepairLifecycleActive(controller),
|
|
1728
1733
|
});
|
|
1729
1734
|
}
|
|
1735
|
+
replicationInfoV2ReceiveCapabilities() {
|
|
1736
|
+
return (SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
1737
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
|
|
1738
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
|
|
1739
|
+
(this._logProperties?.sync?.rawExchangeHeads === true
|
|
1740
|
+
? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
|
|
1741
|
+
: 0));
|
|
1742
|
+
}
|
|
1743
|
+
async advertiseReplicationInfoV2ReceiveCapability(properties) {
|
|
1744
|
+
const peerHash = properties.target.hashcode();
|
|
1745
|
+
const receiverTransportSession = BigInt(this.node.services.pubsub.session);
|
|
1746
|
+
await this.rpc.send(new SyncCapabilitiesMessage({
|
|
1747
|
+
capabilities: this.replicationInfoV2ReceiveCapabilities(),
|
|
1748
|
+
}), {
|
|
1749
|
+
mode: new AcknowledgeDelivery({
|
|
1750
|
+
redundancy: 1,
|
|
1751
|
+
to: [properties.target],
|
|
1752
|
+
}),
|
|
1753
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
1754
|
+
signal: properties.signal,
|
|
1755
|
+
});
|
|
1756
|
+
if (properties.signal.aborted ||
|
|
1757
|
+
this.closed ||
|
|
1758
|
+
!this._peerSessions.isCurrent(peerHash, properties.peerSession) ||
|
|
1759
|
+
properties.peerSession.phase !== "open" ||
|
|
1760
|
+
!properties.peerSession.isActive() ||
|
|
1761
|
+
!this._peerSessions.isReceiveEpochCurrent(peerHash, properties.receiveEpoch) ||
|
|
1762
|
+
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
1763
|
+
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
1764
|
+
BigInt(this.node.services.pubsub.session) !== receiverTransportSession) {
|
|
1765
|
+
return undefined;
|
|
1766
|
+
}
|
|
1767
|
+
return {
|
|
1768
|
+
receiverTransportSession,
|
|
1769
|
+
requestNotBeforeMs: Date.now(),
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
createReplicationInfoV2ReceiveCoordinator() {
|
|
1773
|
+
return new ReplicationInfoV2ReceiveCoordinator({
|
|
1774
|
+
getSelfKey: () => this.node.identity.publicKey,
|
|
1775
|
+
getReceiverTransportSession: () => BigInt(this.node.services.pubsub.session),
|
|
1776
|
+
isClosed: () => this.closed,
|
|
1777
|
+
isPeerSessionCurrent: (peerHash, peerSession) => this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
1778
|
+
peerSession.phase === "open" &&
|
|
1779
|
+
peerSession.isActive(),
|
|
1780
|
+
isReceiveEpochCurrent: (peerHash, receiveEpoch) => this._peerSessions.isReceiveEpochCurrent(peerHash, receiveEpoch),
|
|
1781
|
+
isPeerStateCurrent: (peerHash, peerSession, receiveEpoch) => this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
1782
|
+
peerSession.phase === "open" &&
|
|
1783
|
+
peerSession.isActive() &&
|
|
1784
|
+
this._peerSessions.isReceiveEpochCurrent(peerHash, receiveEpoch) &&
|
|
1785
|
+
!this._peerSessions.isReplicationInfoBlocked(peerHash) &&
|
|
1786
|
+
this._peerSessions.isReceiveCleanupGateOpen(peerHash),
|
|
1787
|
+
isSenderTransportSessionCurrent: (peerHash, senderTransportSession) => this._peerSyncCapabilitySessions.get(peerHash) ===
|
|
1788
|
+
senderTransportSession,
|
|
1789
|
+
sendRequest: async (request, target, signal) => {
|
|
1790
|
+
await this.rpc.send(request, {
|
|
1791
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [target] }),
|
|
1792
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
1793
|
+
signal,
|
|
1794
|
+
});
|
|
1795
|
+
},
|
|
1796
|
+
refreshLocalCapability: (properties) => this.advertiseReplicationInfoV2ReceiveCapability({
|
|
1797
|
+
target: properties.target,
|
|
1798
|
+
peerSession: properties.peerSession,
|
|
1799
|
+
receiveEpoch: properties.receiveEpoch,
|
|
1800
|
+
signal: properties.signal,
|
|
1801
|
+
}),
|
|
1802
|
+
onRequestError: (error) => {
|
|
1803
|
+
if (isNotStartedError(error) ||
|
|
1804
|
+
(this.closed && error instanceof AbortError)) {
|
|
1805
|
+
return;
|
|
1806
|
+
}
|
|
1807
|
+
logger.trace(`Replication-info V2 recovery request failed: ${error?.message ?? String(error)}`);
|
|
1808
|
+
},
|
|
1809
|
+
onLocalCapabilityError: (error) => this.handleReplicationLifecycleSendError(error),
|
|
1810
|
+
});
|
|
1811
|
+
}
|
|
1730
1812
|
createReplicatorLivenessMonitor() {
|
|
1731
1813
|
return new ReplicatorLivenessMonitor({
|
|
1732
1814
|
// Sweep-driven probes and activity marks dispatch through the owner so
|
|
@@ -1831,6 +1913,7 @@ let SharedLog = (() => {
|
|
|
1831
1913
|
this._replicatorJoinEmitted = new Set();
|
|
1832
1914
|
this._replicatorsReconciled = false;
|
|
1833
1915
|
this._liveness = this.createReplicatorLivenessMonitor();
|
|
1916
|
+
this._v2Receive = this.createReplicationInfoV2ReceiveCoordinator();
|
|
1834
1917
|
this._v2Send = this.createReplicationInfoV2SendCoordinator();
|
|
1835
1918
|
this._announcements = this.createReplicationAnnouncementCoordinator();
|
|
1836
1919
|
this.pendingMaturity = new Map();
|
|
@@ -2226,11 +2309,20 @@ let SharedLog = (() => {
|
|
|
2226
2309
|
const { peerHash, capabilities, transportSession, timestamp, openingSession, } = properties;
|
|
2227
2310
|
if (openingSession) {
|
|
2228
2311
|
const previous = this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
2312
|
+
if (transportSession !== undefined &&
|
|
2313
|
+
timestamp !== undefined &&
|
|
2314
|
+
previous?.epoch === openingSession &&
|
|
2315
|
+
previous.timestamp !== undefined &&
|
|
2316
|
+
previous.transportSession !== transportSession &&
|
|
2317
|
+
timestamp <= previous.timestamp) {
|
|
2318
|
+
return false;
|
|
2319
|
+
}
|
|
2229
2320
|
if (transportSession !== undefined &&
|
|
2230
2321
|
timestamp !== undefined &&
|
|
2231
2322
|
previous?.epoch === openingSession &&
|
|
2232
2323
|
previous.transportSession === transportSession) {
|
|
2233
|
-
if (previous.timestamp !== undefined &&
|
|
2324
|
+
if (previous.timestamp !== undefined &&
|
|
2325
|
+
timestamp < previous.timestamp) {
|
|
2234
2326
|
return false;
|
|
2235
2327
|
}
|
|
2236
2328
|
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
@@ -2258,24 +2350,26 @@ let SharedLog = (() => {
|
|
|
2258
2350
|
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
2259
2351
|
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
2260
2352
|
this._v2Send.advancePeerCapability(peerHash);
|
|
2353
|
+
this._v2Receive.revokePeerCapability(peerHash);
|
|
2261
2354
|
return true;
|
|
2262
2355
|
}
|
|
2263
2356
|
const previousSession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
2264
2357
|
const previousTimestamp = this._peerSyncCapabilityTimestamps.get(peerHash);
|
|
2265
2358
|
const sameTransportSession = previousSession === transportSession;
|
|
2266
|
-
if (
|
|
2267
|
-
|
|
2268
|
-
|
|
2359
|
+
if (previousTimestamp !== undefined &&
|
|
2360
|
+
((sameTransportSession && timestamp < previousTimestamp) ||
|
|
2361
|
+
(!sameTransportSession && timestamp <= previousTimestamp))) {
|
|
2269
2362
|
return false;
|
|
2270
2363
|
}
|
|
2271
2364
|
const previousCapabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
|
|
2272
2365
|
const nextCapabilities = sameTransportSession
|
|
2273
2366
|
? previousCapabilities | capabilities
|
|
2274
2367
|
: capabilities;
|
|
2368
|
+
const senderGrantCapabilityMask = SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
2369
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY;
|
|
2275
2370
|
const generationAdvanced = !sameTransportSession ||
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
nextCapabilities !== previousCapabilities;
|
|
2371
|
+
(previousCapabilities & senderGrantCapabilityMask) !==
|
|
2372
|
+
(nextCapabilities & senderGrantCapabilityMask);
|
|
2279
2373
|
this._peerSyncCapabilities.set(peerHash, nextCapabilities);
|
|
2280
2374
|
this._peerSyncCapabilitySessions.set(peerHash, transportSession);
|
|
2281
2375
|
this._peerSyncCapabilityTimestamps.set(peerHash, previousTimestamp === undefined ||
|
|
@@ -2288,6 +2382,24 @@ let SharedLog = (() => {
|
|
|
2288
2382
|
}
|
|
2289
2383
|
return true;
|
|
2290
2384
|
}
|
|
2385
|
+
promoteReplicationInfoV2ReceiveCapability(target, peerSession) {
|
|
2386
|
+
const peerHash = target.hashcode();
|
|
2387
|
+
const senderTransportSession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
2388
|
+
const capabilityTimestamp = this._peerSyncCapabilityTimestamps.get(peerHash);
|
|
2389
|
+
if (senderTransportSession === undefined ||
|
|
2390
|
+
capabilityTimestamp === undefined) {
|
|
2391
|
+
return false;
|
|
2392
|
+
}
|
|
2393
|
+
return this._v2Receive.observeCapability({
|
|
2394
|
+
peerHash,
|
|
2395
|
+
target,
|
|
2396
|
+
peerSession,
|
|
2397
|
+
receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
|
|
2398
|
+
capabilities: this._peerSyncCapabilities.get(peerHash) ?? 0,
|
|
2399
|
+
senderTransportSession,
|
|
2400
|
+
capabilityTimestamp,
|
|
2401
|
+
});
|
|
2402
|
+
}
|
|
2291
2403
|
/**
|
|
2292
2404
|
* Live append gossip may use the raw exchange-heads path only when we
|
|
2293
2405
|
* opted into raw sync and every remote recipient advertised raw capability
|
|
@@ -2984,7 +3096,8 @@ let SharedLog = (() => {
|
|
|
2984
3096
|
return undefined;
|
|
2985
3097
|
}
|
|
2986
3098
|
isReceiveOwnershipSnapshotStable(revision) {
|
|
2987
|
-
return ((this._instanceLifecycle?._receiveOwnershipMutationAdmissions ?? 0) ===
|
|
3099
|
+
return ((this._instanceLifecycle?._receiveOwnershipMutationAdmissions ?? 0) ===
|
|
3100
|
+
0 &&
|
|
2988
3101
|
(this._instanceLifecycle?._receiveOwnershipRevision ?? 0) === revision);
|
|
2989
3102
|
}
|
|
2990
3103
|
canCacheLeaderSelectionContext(options) {
|
|
@@ -3771,6 +3884,7 @@ let SharedLog = (() => {
|
|
|
3771
3884
|
};
|
|
3772
3885
|
let removed = false;
|
|
3773
3886
|
let removalCallCompleted = false;
|
|
3887
|
+
let replicationInfoRecoveryEpochAdvanced = false;
|
|
3774
3888
|
// Replication-info updates already serialize per peer. Put the hash-wide
|
|
3775
3889
|
// removal on the same queue so a newer reset cannot be deleted underneath
|
|
3776
3890
|
// itself by an older unsubscribe callback.
|
|
@@ -3840,6 +3954,13 @@ let SharedLog = (() => {
|
|
|
3840
3954
|
// before mutating ownership.
|
|
3841
3955
|
this.cleanupCheckedPrunePeer(keyHash, replicationOwnershipLifecycleController, checkedPruneCoordinator);
|
|
3842
3956
|
}
|
|
3957
|
+
if (!isMe) {
|
|
3958
|
+
// This is the last synchronous current-generation boundary
|
|
3959
|
+
// before destructive work. Fence parked receives now because
|
|
3960
|
+
// the coherent delete can durably mutate and then fail or poison.
|
|
3961
|
+
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
3962
|
+
replicationInfoRecoveryEpochAdvanced = true;
|
|
3963
|
+
}
|
|
3843
3964
|
const deletion = await this.deleteReplicationRangesCoherently(deleted, keyHash);
|
|
3844
3965
|
deleted = deletion.removed;
|
|
3845
3966
|
ownerHasRanges = deletion.ownerHasRanges;
|
|
@@ -3885,11 +4006,6 @@ let SharedLog = (() => {
|
|
|
3885
4006
|
// through replication-info updates without a topic unsubscribe event.
|
|
3886
4007
|
await cleanupDisconnectedPeer();
|
|
3887
4008
|
if (!isMe) {
|
|
3888
|
-
// Replication-info handlers release their receive lease before joining
|
|
3889
|
-
// this lane. Fence every handler queued behind this successful removal,
|
|
3890
|
-
// regardless of whether it came from liveness, startup pruning, or an
|
|
3891
|
-
// unsubscribe transition.
|
|
3892
|
-
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
3893
4009
|
this.rebalanceParticipationDebounced?.call();
|
|
3894
4010
|
}
|
|
3895
4011
|
removed = true;
|
|
@@ -3920,6 +4036,28 @@ let SharedLog = (() => {
|
|
|
3920
4036
|
}
|
|
3921
4037
|
finally {
|
|
3922
4038
|
releaseReceiveCleanupGate?.();
|
|
4039
|
+
if (replicationInfoRecoveryEpochAdvanced &&
|
|
4040
|
+
ownsReplicationOwnershipLifecycle() &&
|
|
4041
|
+
ownsReplicationLifecycle() &&
|
|
4042
|
+
ownsSubscriptionEpoch()) {
|
|
4043
|
+
// The first recovery arm ran while receive admission was fenced.
|
|
4044
|
+
// Re-arm after releasing the cleanup gate so a still-open peer can
|
|
4045
|
+
// deliver the authoritative Full that completes recovery.
|
|
4046
|
+
const peerSession = this._peerSessions.current(keyHash);
|
|
4047
|
+
if (peerSession?.phase === "open") {
|
|
4048
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(keyHash);
|
|
4049
|
+
this._v2Receive.advanceRecovery({
|
|
4050
|
+
peerHash: keyHash,
|
|
4051
|
+
peerSession,
|
|
4052
|
+
receiveEpoch,
|
|
4053
|
+
});
|
|
4054
|
+
this._v2Receive.reAdvertiseLocalCapabilityForRecovery({
|
|
4055
|
+
peerHash: keyHash,
|
|
4056
|
+
peerSession,
|
|
4057
|
+
receiveEpoch,
|
|
4058
|
+
});
|
|
4059
|
+
}
|
|
4060
|
+
}
|
|
3923
4061
|
if (isSpeculativePeerRemoval) {
|
|
3924
4062
|
const cancelledByFreshActivity = removalCallCompleted &&
|
|
3925
4063
|
!removed &&
|
|
@@ -4012,6 +4150,11 @@ let SharedLog = (() => {
|
|
|
4012
4150
|
}
|
|
4013
4151
|
const deletion = await this.deleteReplicationRangesCoherently(ranges, ownerHash);
|
|
4014
4152
|
ranges = deletion.removed;
|
|
4153
|
+
if ((options?.shouldRemove && !options.shouldRemove()) ||
|
|
4154
|
+
options?.onDurableRemoveCommitted?.() === false) {
|
|
4155
|
+
await deletion.rollback();
|
|
4156
|
+
return false;
|
|
4157
|
+
}
|
|
4015
4158
|
options?.onRemoved?.(ranges);
|
|
4016
4159
|
if (ranges.length > 0) {
|
|
4017
4160
|
this.events.dispatchEvent(new CustomEvent("replication:change", {
|
|
@@ -4075,7 +4218,7 @@ let SharedLog = (() => {
|
|
|
4075
4218
|
this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
|
|
4076
4219
|
return this.withReceiveOwnershipMutationQueue(() => this.addReplicationRangeUnlocked(ranges, from, options, replicationOwnershipLifecycleController), replicationOwnershipLifecycleController);
|
|
4077
4220
|
}
|
|
4078
|
-
async addReplicationRangeUnlocked(ranges, from, { reset, checkDuplicates, timestamp: ts, rebalance, allowLegacyOrderedReplacementPairs, onConfirmedDurableStateChanged, shouldApply, } = {}, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
|
|
4221
|
+
async addReplicationRangeUnlocked(ranges, from, { reset, checkDuplicates, timestamp: ts, rebalance, allowLegacyOrderedReplacementPairs, onConfirmedDurableStateChanged, onDurableApplyCommitted, shouldApply, } = {}, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
|
|
4079
4222
|
this.validateReplicationRangeAnnouncement(ranges);
|
|
4080
4223
|
this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
|
|
4081
4224
|
// This predicate is intentionally checked inside the global ownership lane.
|
|
@@ -4084,6 +4227,12 @@ let SharedLog = (() => {
|
|
|
4084
4227
|
if (shouldApply && !shouldApply()) {
|
|
4085
4228
|
return [];
|
|
4086
4229
|
}
|
|
4230
|
+
const applySuperseded = Symbol("replication-range-apply-superseded");
|
|
4231
|
+
const throwIfApplySuperseded = () => {
|
|
4232
|
+
if (shouldApply && !shouldApply()) {
|
|
4233
|
+
throw applySuperseded;
|
|
4234
|
+
}
|
|
4235
|
+
};
|
|
4087
4236
|
const fromHash = from.hashcode();
|
|
4088
4237
|
const incomingRangesById = new Map();
|
|
4089
4238
|
const incomingRangeCountsById = new Map();
|
|
@@ -4142,6 +4291,7 @@ let SharedLog = (() => {
|
|
|
4142
4291
|
let isStoppedReplicating = false;
|
|
4143
4292
|
let wasReplicatorBeforeDestructiveReset = false;
|
|
4144
4293
|
let resetFailureLeaveEmitted = false;
|
|
4294
|
+
let resetDeletionRollback;
|
|
4145
4295
|
const publishConfirmedResetStop = (ownerHasRanges) => {
|
|
4146
4296
|
if (ownerHasRanges || resetFailureLeaveEmitted) {
|
|
4147
4297
|
return;
|
|
@@ -4182,8 +4332,14 @@ let SharedLog = (() => {
|
|
|
4182
4332
|
else {
|
|
4183
4333
|
wasReplicatorBeforeDestructiveReset =
|
|
4184
4334
|
this.uniqueReplicators.has(fromHash);
|
|
4335
|
+
throwIfApplySuperseded();
|
|
4185
4336
|
const deletion = await this.deleteReplicationRangesCoherently(deleted, fromHash, { preserveOwnerMembership: ranges.length > 0 });
|
|
4186
4337
|
deleted = deletion.removed;
|
|
4338
|
+
resetDeletionRollback = deletion.rollback;
|
|
4339
|
+
if (shouldApply && !shouldApply()) {
|
|
4340
|
+
await deletion.rollback();
|
|
4341
|
+
return [];
|
|
4342
|
+
}
|
|
4187
4343
|
diffs = [
|
|
4188
4344
|
...deleted.map((x) => {
|
|
4189
4345
|
return { range: x, type: "removed", timestamp };
|
|
@@ -4339,19 +4495,27 @@ let SharedLog = (() => {
|
|
|
4339
4495
|
return failure;
|
|
4340
4496
|
};
|
|
4341
4497
|
try {
|
|
4498
|
+
throwIfApplySuperseded();
|
|
4342
4499
|
for (const diff of diffs) {
|
|
4343
4500
|
if (diff.type !== "added") {
|
|
4344
4501
|
continue;
|
|
4345
4502
|
}
|
|
4503
|
+
throwIfApplySuperseded();
|
|
4346
4504
|
appliedPositiveRanges.push(diff);
|
|
4347
4505
|
await this.replicationIndex.put(diff.range);
|
|
4348
4506
|
this.putNativeReplicationRange(diff.range);
|
|
4507
|
+
throwIfApplySuperseded();
|
|
4349
4508
|
}
|
|
4350
4509
|
if (reset && diffs.length > 0) {
|
|
4351
4510
|
await this.updateOldestTimestampFromIndex();
|
|
4511
|
+
throwIfApplySuperseded();
|
|
4512
|
+
}
|
|
4513
|
+
if (onDurableApplyCommitted?.() === false) {
|
|
4514
|
+
throw applySuperseded;
|
|
4352
4515
|
}
|
|
4353
4516
|
}
|
|
4354
4517
|
catch (error) {
|
|
4518
|
+
const superseded = error === applySuperseded;
|
|
4355
4519
|
let outcomeError = error;
|
|
4356
4520
|
if (appliedPositiveRanges.length > 0) {
|
|
4357
4521
|
try {
|
|
@@ -4361,6 +4525,20 @@ let SharedLog = (() => {
|
|
|
4361
4525
|
outcomeError = poisonFromPositiveRollback(rollbackError, error);
|
|
4362
4526
|
}
|
|
4363
4527
|
}
|
|
4528
|
+
if (superseded) {
|
|
4529
|
+
if (resetDeletionRollback) {
|
|
4530
|
+
try {
|
|
4531
|
+
await resetDeletionRollback();
|
|
4532
|
+
}
|
|
4533
|
+
catch (rollbackError) {
|
|
4534
|
+
outcomeError = rollbackError;
|
|
4535
|
+
}
|
|
4536
|
+
}
|
|
4537
|
+
if (outcomeError === applySuperseded) {
|
|
4538
|
+
return [];
|
|
4539
|
+
}
|
|
4540
|
+
throw outcomeError;
|
|
4541
|
+
}
|
|
4364
4542
|
if (reset) {
|
|
4365
4543
|
const negativeDiffs = diffs.filter((diff) => diff.type !== "added");
|
|
4366
4544
|
if (negativeDiffs.length > 0) {
|
|
@@ -6967,7 +7145,8 @@ let SharedLog = (() => {
|
|
|
6967
7145
|
appendLocallyPreparedPayloadNativeBackboneStorageTransaction(payloadData, appendOptions, properties, minReplicasValue, runtimeOnlyCoordinates, ownershipLifecycleController) {
|
|
6968
7146
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
6969
7147
|
const backbone = this._nativeBackbone;
|
|
6970
|
-
if (!backbone ||
|
|
7148
|
+
if (!backbone ||
|
|
7149
|
+
!this._coordinates.canUseNativeBackboneResidentCoordinateState()) {
|
|
6971
7150
|
return undefined;
|
|
6972
7151
|
}
|
|
6973
7152
|
if (properties?.nativeBackboneDocumentDeleteKey &&
|
|
@@ -7098,11 +7277,12 @@ let SharedLog = (() => {
|
|
|
7098
7277
|
backboneAppend.trimmed.map((entry) => entry.hash);
|
|
7099
7278
|
const committedHash = backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
7100
7279
|
const committedNext = backboneAppend.entry.next ?? next;
|
|
7101
|
-
nativeCoordinateRollback =
|
|
7102
|
-
|
|
7103
|
-
|
|
7104
|
-
|
|
7105
|
-
|
|
7280
|
+
nativeCoordinateRollback =
|
|
7281
|
+
this._coordinates.snapshotResidentCoordinateEntries([
|
|
7282
|
+
...(committedHash ? [committedHash] : []),
|
|
7283
|
+
...committedNext,
|
|
7284
|
+
...nativeTrimmedHashes,
|
|
7285
|
+
]);
|
|
7106
7286
|
await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, committedHash ? [committedHash] : [], nativeTrimmedHashes, nativeCoordinateRollback, combineCoordinateDeleteHashes(committedNext, nativeTrimmedHashes));
|
|
7107
7287
|
const rollbackCommitted = async (cause, committedCids) => {
|
|
7108
7288
|
durableWrapper?.cancelNativeDeleteCleanup?.(nativeDeleteCleanupToken);
|
|
@@ -7770,14 +7950,15 @@ let SharedLog = (() => {
|
|
|
7770
7950
|
const nativeTrimmedHashSet = new Set(committedAppends.flatMap((append) => append.trimmedHashes ??
|
|
7771
7951
|
append.trimmed.map((entry) => entry.hash)));
|
|
7772
7952
|
const nativeTrimmedHashes = [...nativeTrimmedHashSet];
|
|
7773
|
-
batchCoordinateRollback =
|
|
7774
|
-
|
|
7775
|
-
|
|
7776
|
-
|
|
7777
|
-
|
|
7778
|
-
|
|
7779
|
-
append.
|
|
7780
|
-
|
|
7953
|
+
batchCoordinateRollback =
|
|
7954
|
+
this._coordinates.snapshotResidentCoordinateEntries(committedAppends.flatMap((append) => [
|
|
7955
|
+
...((append.entry.cid ?? append.entry.hash)
|
|
7956
|
+
? [append.entry.cid ?? append.entry.hash]
|
|
7957
|
+
: []),
|
|
7958
|
+
...append.entry.next,
|
|
7959
|
+
...(append.trimmedHashes ??
|
|
7960
|
+
append.trimmed.map((entry) => entry.hash)),
|
|
7961
|
+
]));
|
|
7781
7962
|
await this.setNativeStrictDurableTransactionOperation(nativeStrictTransaction, committedCids, nativeTrimmedHashes, batchCoordinateRollback, committedAppends.flatMap((append) => [
|
|
7782
7963
|
...append.entry.next,
|
|
7783
7964
|
...(append.trimmedHashes ??
|
|
@@ -9004,6 +9185,8 @@ let SharedLog = (() => {
|
|
|
9004
9185
|
this._lastLocalAppendAt = 0;
|
|
9005
9186
|
this._announcements ??= this.createReplicationAnnouncementCoordinator();
|
|
9006
9187
|
this._announcements.resetForOpen();
|
|
9188
|
+
this._v2Receive ??= this.createReplicationInfoV2ReceiveCoordinator();
|
|
9189
|
+
this._v2Receive.resetForOpen();
|
|
9007
9190
|
this._v2Send ??= this.createReplicationInfoV2SendCoordinator();
|
|
9008
9191
|
this._v2Send.resetForOpen();
|
|
9009
9192
|
const adaptiveReplicateOptions = options?.replicate && isAdaptiveReplicatorOption(options.replicate)
|
|
@@ -9575,7 +9758,8 @@ let SharedLog = (() => {
|
|
|
9575
9758
|
// any stale coordinate or document state to this backbone.
|
|
9576
9759
|
await this._coordinates._nativeBackboneCoordinatePersistence.resumeDrop?.();
|
|
9577
9760
|
await this._coordinates._nativeBackboneCoordinatePersistence.hydrate(backbone);
|
|
9578
|
-
this._coordinates._nativeBackboneCoordinateJournalLastFlushMs =
|
|
9761
|
+
this._coordinates._nativeBackboneCoordinateJournalLastFlushMs =
|
|
9762
|
+
Date.now();
|
|
9579
9763
|
this.hydrateNativeCoordinateStateFromBackbone(backbone);
|
|
9580
9764
|
return;
|
|
9581
9765
|
}
|
|
@@ -9787,18 +9971,20 @@ let SharedLog = (() => {
|
|
|
9787
9971
|
}
|
|
9788
9972
|
if (this.node.directory != null &&
|
|
9789
9973
|
this._coordinates._nativeBackboneCoordinatePersistence) {
|
|
9790
|
-
if (this._coordinates._nativeBackboneCoordinatePersistence
|
|
9974
|
+
if (this._coordinates._nativeBackboneCoordinatePersistence
|
|
9975
|
+
.durableBarrier !== true ||
|
|
9791
9976
|
typeof this._nativeBackboneCoordinatePersistenceStore
|
|
9792
9977
|
?.durableBarrier !== "function") {
|
|
9793
9978
|
throw new Error("Durable nativeBackbone coordinate persistence requires an explicit physical durability barrier");
|
|
9794
9979
|
}
|
|
9795
9980
|
}
|
|
9796
9981
|
if (this._coordinates._nativeBackboneCoordinatePersistence &&
|
|
9797
|
-
(this._coordinates._nativeBackboneCoordinatePersistence
|
|
9798
|
-
null ||
|
|
9799
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
9800
|
-
null) &&
|
|
9801
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
9982
|
+
(this._coordinates._nativeBackboneCoordinatePersistence
|
|
9983
|
+
.compactMaxJournalBytes != null ||
|
|
9984
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
9985
|
+
.compactMaxJournalRecords != null) &&
|
|
9986
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
9987
|
+
.crashSafeCompaction !== true) {
|
|
9802
9988
|
// Durable custom adapters must explicitly advertise an atomic generation
|
|
9803
9989
|
// protocol before SharedLog permits automatic WAL compaction.
|
|
9804
9990
|
throw new Error("Durable native coordinate persistence compaction thresholds require crashSafeCompaction");
|
|
@@ -10121,13 +10307,18 @@ let SharedLog = (() => {
|
|
|
10121
10307
|
}
|
|
10122
10308
|
}
|
|
10123
10309
|
cleanupPeerDisconnectTracking(peerHash, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
|
|
10310
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
10311
|
+
const preserveV2Session = peerSession?.phase === "open" && peerSession.isActive();
|
|
10124
10312
|
this.cancelReplicationInfoRequests(peerHash);
|
|
10125
10313
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
10126
10314
|
this._liveness._replicatorLastActivityAt.delete(peerHash);
|
|
10127
|
-
|
|
10128
|
-
|
|
10129
|
-
|
|
10130
|
-
|
|
10315
|
+
if (!preserveV2Session) {
|
|
10316
|
+
this._peerSyncCapabilities.delete(peerHash);
|
|
10317
|
+
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
10318
|
+
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
10319
|
+
this._v2Receive.clearPeer(peerHash);
|
|
10320
|
+
this._v2Send.clearPeer(peerHash);
|
|
10321
|
+
}
|
|
10131
10322
|
this.cleanupPendingIHavePeer(peerHash);
|
|
10132
10323
|
this.cleanupCheckedPrunePeer(peerHash, ownershipLifecycleController, this._checkedPrune);
|
|
10133
10324
|
}
|
|
@@ -10145,8 +10336,16 @@ let SharedLog = (() => {
|
|
|
10145
10336
|
// when they eventually reach the apply lane. Reset the sender's
|
|
10146
10337
|
// ordering watermark with the local epoch so a later arrival can be
|
|
10147
10338
|
// accepted without comparing its clock to this receiver's clock.
|
|
10148
|
-
this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
10339
|
+
const receiveEpoch = this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
10149
10340
|
this.latestReplicationInfoMessage.delete(peerHash);
|
|
10341
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
10342
|
+
if (peerSession?.phase === "open") {
|
|
10343
|
+
this._v2Receive.advanceRecovery({
|
|
10344
|
+
peerHash,
|
|
10345
|
+
peerSession,
|
|
10346
|
+
receiveEpoch,
|
|
10347
|
+
});
|
|
10348
|
+
}
|
|
10150
10349
|
}
|
|
10151
10350
|
async resolveCandidatePeersForHash(hash, options) {
|
|
10152
10351
|
if (options?.signal?.aborted)
|
|
@@ -10946,6 +11145,7 @@ let SharedLog = (() => {
|
|
|
10946
11145
|
this._peerSyncCapabilities?.clear();
|
|
10947
11146
|
this._peerSyncCapabilitySessions?.clear();
|
|
10948
11147
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
11148
|
+
this._v2Receive?.clearForClose();
|
|
10949
11149
|
this._v2Send?.clearForClose();
|
|
10950
11150
|
this._liveRawGossipBatches?.clear();
|
|
10951
11151
|
this._nativeSharedLogState?.clearGidPeers();
|
|
@@ -11009,6 +11209,7 @@ let SharedLog = (() => {
|
|
|
11009
11209
|
this.preventParentAttachments();
|
|
11010
11210
|
this.stopRepairLifecycle();
|
|
11011
11211
|
this._instanceLifecycle?.beginTerminal("close");
|
|
11212
|
+
this._v2Receive?.clearForClose();
|
|
11012
11213
|
const replicationRangeTerminalFence = this.acquireReplicationRangeMutationTerminalFence();
|
|
11013
11214
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
11014
11215
|
try {
|
|
@@ -11069,10 +11270,12 @@ let SharedLog = (() => {
|
|
|
11069
11270
|
try {
|
|
11070
11271
|
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
11071
11272
|
await Promise.all([
|
|
11072
|
-
this.rpc
|
|
11273
|
+
this.rpc
|
|
11274
|
+
.send(reset, {
|
|
11073
11275
|
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
11074
11276
|
signal: abort.signal,
|
|
11075
|
-
})
|
|
11277
|
+
})
|
|
11278
|
+
.catch(() => { }),
|
|
11076
11279
|
this._v2Send.sendTerminalReset(abort.signal),
|
|
11077
11280
|
]);
|
|
11078
11281
|
}
|
|
@@ -11151,6 +11354,7 @@ let SharedLog = (() => {
|
|
|
11151
11354
|
this.preventParentAttachments();
|
|
11152
11355
|
this.stopRepairLifecycle();
|
|
11153
11356
|
this._instanceLifecycle?.beginTerminal("drop");
|
|
11357
|
+
this._v2Receive?.clearForClose();
|
|
11154
11358
|
const replicationRangeTerminalFence = this.acquireReplicationRangeMutationTerminalFence();
|
|
11155
11359
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
11156
11360
|
try {
|
|
@@ -11194,10 +11398,12 @@ let SharedLog = (() => {
|
|
|
11194
11398
|
try {
|
|
11195
11399
|
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
11196
11400
|
await Promise.all([
|
|
11197
|
-
this.rpc
|
|
11401
|
+
this.rpc
|
|
11402
|
+
.send(reset, {
|
|
11198
11403
|
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
11199
11404
|
signal: abort.signal,
|
|
11200
|
-
})
|
|
11405
|
+
})
|
|
11406
|
+
.catch(() => { }),
|
|
11201
11407
|
this._v2Send.sendTerminalReset(abort.signal),
|
|
11202
11408
|
]);
|
|
11203
11409
|
}
|
|
@@ -11614,13 +11820,6 @@ let SharedLog = (() => {
|
|
|
11614
11820
|
}
|
|
11615
11821
|
// Callback for receiving a message from the network
|
|
11616
11822
|
async onMessage(msg, context) {
|
|
11617
|
-
// Decode-first only: capability advertisement is not permission to use
|
|
11618
|
-
// V2. Drop every unsolicited V2 envelope before even entering the shared
|
|
11619
|
-
// receive envelope, so durable poison checks, leases, liveness, ordering
|
|
11620
|
-
// watermarks, replication ranges and cleanup side effects cannot observe it.
|
|
11621
|
-
if (isReplicationInfoV2Message(msg)) {
|
|
11622
|
-
return;
|
|
11623
|
-
}
|
|
11624
11823
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
11625
11824
|
? msg
|
|
11626
11825
|
: undefined;
|
|
@@ -11661,18 +11860,25 @@ let SharedLog = (() => {
|
|
|
11661
11860
|
msg = msg.toReplicationInfoMessage(); // migration
|
|
11662
11861
|
}
|
|
11663
11862
|
if (msg instanceof AllReplicatingSegmentsMessage ||
|
|
11664
|
-
msg instanceof AddedReplicationSegmentMessage
|
|
11863
|
+
msg instanceof AddedReplicationSegmentMessage ||
|
|
11864
|
+
msg instanceof FullReplicationInfoV2Message ||
|
|
11865
|
+
msg instanceof AddedReplicationInfoV2Message) {
|
|
11665
11866
|
// Bound decoded untrusted vectors before per-peer/global mutation
|
|
11666
11867
|
// queues, trusted-replicator authorization, or liveness side effects.
|
|
11667
11868
|
this.validateReplicationRangeAnnouncement(msg.segments);
|
|
11668
11869
|
}
|
|
11669
|
-
else if (msg instanceof StoppedReplicating
|
|
11870
|
+
else if (msg instanceof StoppedReplicating ||
|
|
11871
|
+
msg instanceof StoppedReplicationInfoV2Message) {
|
|
11670
11872
|
// Bound the raw decoded vector before deduplication can hide the
|
|
11671
11873
|
// allocation cost, and before liveness or apply-queue side effects.
|
|
11672
11874
|
this.validateStoppedReplicationAnnouncement(msg.segmentIds);
|
|
11673
11875
|
}
|
|
11674
11876
|
if (!context.from.equals(this.node.identity.publicKey) &&
|
|
11675
|
-
!(msg instanceof RequestReplicationInfoV2Message)
|
|
11877
|
+
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
11878
|
+
!isReplicationInfoV2Message(msg) &&
|
|
11879
|
+
!(msg instanceof AllReplicatingSegmentsMessage) &&
|
|
11880
|
+
!(msg instanceof AddedReplicationSegmentMessage) &&
|
|
11881
|
+
!(msg instanceof StoppedReplicating)) {
|
|
11676
11882
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
11677
11883
|
}
|
|
11678
11884
|
const syncProfile = this._logProperties?.sync?.profile;
|
|
@@ -12988,6 +13194,7 @@ let SharedLog = (() => {
|
|
|
12988
13194
|
fromHash: receiveFromHash,
|
|
12989
13195
|
session: receiveSession,
|
|
12990
13196
|
lifecycleController: receiveReplicationLifecycleController,
|
|
13197
|
+
ownershipLifecycleController: receiveOwnershipLifecycleController,
|
|
12991
13198
|
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
12992
13199
|
syncProfile,
|
|
12993
13200
|
lease: peerReceiveLease,
|
|
@@ -13013,6 +13220,9 @@ let SharedLog = (() => {
|
|
|
13013
13220
|
this.clearRepairFrontierHashes(context.from.hashcode(), msg.hashes);
|
|
13014
13221
|
return;
|
|
13015
13222
|
}
|
|
13223
|
+
else if (isReplicationInfoV2Message(msg)) {
|
|
13224
|
+
await this.handleReplicationInfoV2Announcement(msg, laneRequestContext, lane);
|
|
13225
|
+
}
|
|
13016
13226
|
else if (msg instanceof SyncCapabilitiesMessage) {
|
|
13017
13227
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
13018
13228
|
const capabilityTransportSession = context.message?.header?.session;
|
|
@@ -13033,13 +13243,14 @@ let SharedLog = (() => {
|
|
|
13033
13243
|
openingSession: receiveSession,
|
|
13034
13244
|
});
|
|
13035
13245
|
}
|
|
13036
|
-
else {
|
|
13037
|
-
|
|
13038
|
-
|
|
13039
|
-
|
|
13040
|
-
|
|
13041
|
-
|
|
13042
|
-
|
|
13246
|
+
else if (this.observePeerSyncCapabilities({
|
|
13247
|
+
peerHash: receiveFromHash,
|
|
13248
|
+
capabilities: msg.capabilities,
|
|
13249
|
+
transportSession: capabilityTransportSession,
|
|
13250
|
+
timestamp: capabilityTimestamp,
|
|
13251
|
+
}) &&
|
|
13252
|
+
receiveSession?.phase === "open") {
|
|
13253
|
+
this.promoteReplicationInfoV2ReceiveCapability(context.from, receiveSession);
|
|
13043
13254
|
}
|
|
13044
13255
|
}
|
|
13045
13256
|
return;
|
|
@@ -13060,6 +13271,7 @@ let SharedLog = (() => {
|
|
|
13060
13271
|
from: context.from,
|
|
13061
13272
|
peerSession: receiveSession,
|
|
13062
13273
|
receiverTransportSession: context.message.header.session,
|
|
13274
|
+
capabilityTimestamp: this._peerSyncCapabilityTimestamps.get(receiveFromHash),
|
|
13063
13275
|
requestTimestamp: context.message.header.timestamp,
|
|
13064
13276
|
})) {
|
|
13065
13277
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
@@ -13335,6 +13547,147 @@ let SharedLog = (() => {
|
|
|
13335
13547
|
}
|
|
13336
13548
|
}
|
|
13337
13549
|
}
|
|
13550
|
+
async handleReplicationInfoV2Announcement(msg, context, lane) {
|
|
13551
|
+
const from = context.from;
|
|
13552
|
+
const fromHash = lane.fromHash;
|
|
13553
|
+
const receiveSession = lane.session;
|
|
13554
|
+
if (from.equals(this.node.identity.publicKey) ||
|
|
13555
|
+
receiveSession === null ||
|
|
13556
|
+
receiveSession.phase !== "open") {
|
|
13557
|
+
return;
|
|
13558
|
+
}
|
|
13559
|
+
// Authenticate and reserve before the per-peer lane. One reservation per
|
|
13560
|
+
// peer bounds decoded-frame retention while another mutation is parked.
|
|
13561
|
+
const receiveState = this._v2Receive._receiveStates.get(fromHash);
|
|
13562
|
+
if (!receiveState ||
|
|
13563
|
+
receiveState.peerSession !== receiveSession ||
|
|
13564
|
+
receiveState.senderTransportSession !== context.message.header.session) {
|
|
13565
|
+
return;
|
|
13566
|
+
}
|
|
13567
|
+
const admission = this._v2Receive.reserve(msg, {
|
|
13568
|
+
from,
|
|
13569
|
+
peerSession: receiveSession,
|
|
13570
|
+
receiveEpoch: lane.receiveEpoch,
|
|
13571
|
+
senderTransportSession: context.message.header.session,
|
|
13572
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13573
|
+
});
|
|
13574
|
+
if (!admission) {
|
|
13575
|
+
return;
|
|
13576
|
+
}
|
|
13577
|
+
lane.lease.release();
|
|
13578
|
+
try {
|
|
13579
|
+
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
13580
|
+
const hostGate = () => this._instanceLifecycle.isMembershipActiveFor(lane.lifecycleController) &&
|
|
13581
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController) &&
|
|
13582
|
+
this._peerSessions.isCurrent(fromHash, receiveSession) &&
|
|
13583
|
+
receiveSession.phase === "open" &&
|
|
13584
|
+
this._peerSessions.isReceiveEpochCurrent(fromHash, lane.receiveEpoch) &&
|
|
13585
|
+
!this._peerSessions.isReplicationInfoBlocked(fromHash) &&
|
|
13586
|
+
this._peerSessions.isReceiveCleanupGateOpen(fromHash);
|
|
13587
|
+
if (!hostGate()) {
|
|
13588
|
+
return;
|
|
13589
|
+
}
|
|
13590
|
+
const exactGate = () => hostGate() && this._v2Receive.isAdmissionCurrent(admission);
|
|
13591
|
+
let durableCommitted = false;
|
|
13592
|
+
try {
|
|
13593
|
+
if (msg instanceof FullReplicationInfoV2Message ||
|
|
13594
|
+
msg instanceof AddedReplicationInfoV2Message) {
|
|
13595
|
+
let mutationGateChecked = false;
|
|
13596
|
+
let mutationGateAdmitted = false;
|
|
13597
|
+
const result = await this.addReplicationRange(msg.segments.map((segment) => segment.toReplicationRangeIndexable(from)), from, {
|
|
13598
|
+
reset: msg instanceof FullReplicationInfoV2Message,
|
|
13599
|
+
checkDuplicates: true,
|
|
13600
|
+
timestamp: Number(context.message.header.timestamp),
|
|
13601
|
+
allowLegacyOrderedReplacementPairs: msg instanceof AddedReplicationInfoV2Message,
|
|
13602
|
+
shouldApply: () => {
|
|
13603
|
+
mutationGateChecked = true;
|
|
13604
|
+
mutationGateAdmitted = exactGate();
|
|
13605
|
+
return mutationGateAdmitted;
|
|
13606
|
+
},
|
|
13607
|
+
onDurableApplyCommitted: () => {
|
|
13608
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
13609
|
+
return false;
|
|
13610
|
+
}
|
|
13611
|
+
durableCommitted = true;
|
|
13612
|
+
return true;
|
|
13613
|
+
},
|
|
13614
|
+
}, lane.ownershipLifecycleController);
|
|
13615
|
+
if (result === undefined ||
|
|
13616
|
+
!mutationGateChecked ||
|
|
13617
|
+
!mutationGateAdmitted ||
|
|
13618
|
+
!durableCommitted ||
|
|
13619
|
+
!exactGate()) {
|
|
13620
|
+
return;
|
|
13621
|
+
}
|
|
13622
|
+
}
|
|
13623
|
+
else {
|
|
13624
|
+
if (!exactGate()) {
|
|
13625
|
+
return;
|
|
13626
|
+
}
|
|
13627
|
+
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(msg.segmentIds, from);
|
|
13628
|
+
if (!exactGate()) {
|
|
13629
|
+
return;
|
|
13630
|
+
}
|
|
13631
|
+
let mutationGateAdmitted = true;
|
|
13632
|
+
const removedRanges = [];
|
|
13633
|
+
const removed = await this.removeReplicationRanges(rangesToRemove, from, {
|
|
13634
|
+
shouldRemove: () => {
|
|
13635
|
+
mutationGateAdmitted = exactGate();
|
|
13636
|
+
return mutationGateAdmitted;
|
|
13637
|
+
},
|
|
13638
|
+
onDurableRemoveCommitted: () => {
|
|
13639
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
13640
|
+
return false;
|
|
13641
|
+
}
|
|
13642
|
+
durableCommitted = true;
|
|
13643
|
+
return true;
|
|
13644
|
+
},
|
|
13645
|
+
onRemoved: (ranges) => removedRanges.push(...ranges),
|
|
13646
|
+
}, lane.ownershipLifecycleController);
|
|
13647
|
+
if (!durableCommitted) {
|
|
13648
|
+
if (!mutationGateAdmitted ||
|
|
13649
|
+
!exactGate() ||
|
|
13650
|
+
removed ||
|
|
13651
|
+
!this._v2Receive.commit(admission)) {
|
|
13652
|
+
return;
|
|
13653
|
+
}
|
|
13654
|
+
durableCommitted = true;
|
|
13655
|
+
}
|
|
13656
|
+
if (this._instanceLifecycle.isMembershipActiveFor(lane.lifecycleController) &&
|
|
13657
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController)) {
|
|
13658
|
+
const timestamp = BigInt(Date.now());
|
|
13659
|
+
for (const range of removedRanges) {
|
|
13660
|
+
this.replicationChangeDebounceFn.add({
|
|
13661
|
+
range,
|
|
13662
|
+
type: "removed",
|
|
13663
|
+
timestamp,
|
|
13664
|
+
});
|
|
13665
|
+
}
|
|
13666
|
+
}
|
|
13667
|
+
}
|
|
13668
|
+
}
|
|
13669
|
+
catch (error) {
|
|
13670
|
+
this._v2Receive.requireFullAfterFailure(admission);
|
|
13671
|
+
throw error;
|
|
13672
|
+
}
|
|
13673
|
+
if (!durableCommitted) {
|
|
13674
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
13675
|
+
return;
|
|
13676
|
+
}
|
|
13677
|
+
}
|
|
13678
|
+
if (!hostGate()) {
|
|
13679
|
+
return;
|
|
13680
|
+
}
|
|
13681
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
13682
|
+
if (msg instanceof FullReplicationInfoV2Message) {
|
|
13683
|
+
this.cancelReplicationInfoRequests(fromHash);
|
|
13684
|
+
}
|
|
13685
|
+
});
|
|
13686
|
+
}
|
|
13687
|
+
finally {
|
|
13688
|
+
this._v2Receive.release(admission);
|
|
13689
|
+
}
|
|
13690
|
+
}
|
|
13338
13691
|
async handleReplicationInfoAnnouncement(msg, context, lane) {
|
|
13339
13692
|
const receiveFromHash = lane.fromHash;
|
|
13340
13693
|
const receiveSession = lane.session;
|
|
@@ -13350,6 +13703,19 @@ let SharedLog = (() => {
|
|
|
13350
13703
|
// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
|
|
13351
13704
|
const from = context.from;
|
|
13352
13705
|
const fromHash = from.hashcode();
|
|
13706
|
+
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
13707
|
+
if (receiveSession) {
|
|
13708
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
13709
|
+
peerHash: fromHash,
|
|
13710
|
+
peerSession: receiveSession,
|
|
13711
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
13712
|
+
senderTransportSession: context.message.header.session,
|
|
13713
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13714
|
+
message: msg,
|
|
13715
|
+
});
|
|
13716
|
+
}
|
|
13717
|
+
return;
|
|
13718
|
+
}
|
|
13353
13719
|
// Pre-lane gate: lifecycle -> receive-epoch -> blocked, exactly the
|
|
13354
13720
|
// legacy order. isMembershipActiveFor is the unit-pinned fold of
|
|
13355
13721
|
// isReplicationLifecycleActive; isReceiveEpochCurrent is the
|
|
@@ -13374,6 +13740,17 @@ let SharedLog = (() => {
|
|
|
13374
13740
|
this._peerSessions.isReplicationInfoBlocked(fromHash)) {
|
|
13375
13741
|
return;
|
|
13376
13742
|
}
|
|
13743
|
+
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
13744
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
13745
|
+
peerHash: fromHash,
|
|
13746
|
+
peerSession: receiveSession,
|
|
13747
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
13748
|
+
senderTransportSession: context.message.header.session,
|
|
13749
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13750
|
+
message: msg,
|
|
13751
|
+
});
|
|
13752
|
+
return;
|
|
13753
|
+
}
|
|
13377
13754
|
// Process in-order to avoid races where repeated reset messages arrive
|
|
13378
13755
|
// concurrently and trigger spurious "added" diffs / rebalancing.
|
|
13379
13756
|
const prev = this.latestReplicationInfoMessage.get(fromHash);
|
|
@@ -13381,17 +13758,20 @@ let SharedLog = (() => {
|
|
|
13381
13758
|
return;
|
|
13382
13759
|
}
|
|
13383
13760
|
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
13384
|
-
this._liveness._replicatorLivenessFailures.delete(fromHash);
|
|
13385
13761
|
if (this.closed) {
|
|
13386
13762
|
return;
|
|
13387
13763
|
}
|
|
13388
13764
|
const reset = msg instanceof AllReplicatingSegmentsMessage;
|
|
13389
|
-
await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(from)), from, {
|
|
13765
|
+
const result = await this.addReplicationRange(replicationInfoMessage.segments.map((x) => x.toReplicationRangeIndexable(from)), from, {
|
|
13390
13766
|
reset,
|
|
13391
13767
|
checkDuplicates: true,
|
|
13392
13768
|
timestamp: Number(messageTimestamp),
|
|
13393
13769
|
allowLegacyOrderedReplacementPairs: msg instanceof AddedReplicationSegmentMessage,
|
|
13394
13770
|
});
|
|
13771
|
+
if (result === undefined) {
|
|
13772
|
+
return;
|
|
13773
|
+
}
|
|
13774
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
13395
13775
|
// If the peer reports any replication segments, stop re-requesting.
|
|
13396
13776
|
// (Empty reports can be transient during startup.)
|
|
13397
13777
|
if (replicationInfoMessage.segments.length > 0) {
|
|
@@ -13418,6 +13798,19 @@ let SharedLog = (() => {
|
|
|
13418
13798
|
return;
|
|
13419
13799
|
}
|
|
13420
13800
|
const fromHash = from.hashcode();
|
|
13801
|
+
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
13802
|
+
if (receiveSession) {
|
|
13803
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
13804
|
+
peerHash: fromHash,
|
|
13805
|
+
peerSession: receiveSession,
|
|
13806
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
13807
|
+
senderTransportSession: context.message.header.session,
|
|
13808
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13809
|
+
message: msg,
|
|
13810
|
+
});
|
|
13811
|
+
}
|
|
13812
|
+
return;
|
|
13813
|
+
}
|
|
13421
13814
|
// Same pre-lane gate shape as Added/All above (and the same
|
|
13422
13815
|
// intentional absence of a subscription-epoch term).
|
|
13423
13816
|
if (!this._instanceLifecycle.isMembershipActiveFor(receiveReplicationLifecycleController) ||
|
|
@@ -13434,17 +13827,28 @@ let SharedLog = (() => {
|
|
|
13434
13827
|
this._peerSessions.isReplicationInfoBlocked(fromHash)) {
|
|
13435
13828
|
return;
|
|
13436
13829
|
}
|
|
13830
|
+
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
13831
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
13832
|
+
peerHash: fromHash,
|
|
13833
|
+
peerSession: receiveSession,
|
|
13834
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
13835
|
+
senderTransportSession: context.message.header.session,
|
|
13836
|
+
transportTimestamp: context.message.header.timestamp,
|
|
13837
|
+
message: msg,
|
|
13838
|
+
});
|
|
13839
|
+
return;
|
|
13840
|
+
}
|
|
13437
13841
|
const previousTimestamp = this.latestReplicationInfoMessage.get(fromHash);
|
|
13438
13842
|
if (previousTimestamp && previousTimestamp > messageTimestamp) {
|
|
13439
13843
|
return;
|
|
13440
13844
|
}
|
|
13441
13845
|
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
13442
|
-
this._liveness._replicatorLivenessFailures.delete(fromHash);
|
|
13443
13846
|
if (this.closed) {
|
|
13444
13847
|
return;
|
|
13445
13848
|
}
|
|
13446
13849
|
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(segmentIds, from);
|
|
13447
13850
|
await this.removeReplicationRanges(rangesToRemove, from);
|
|
13851
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
13448
13852
|
const timestamp = BigInt(+new Date());
|
|
13449
13853
|
for (const range of rangesToRemove) {
|
|
13450
13854
|
this.replicationChangeDebounceFn.add({
|
|
@@ -15313,6 +15717,8 @@ let SharedLog = (() => {
|
|
|
15313
15717
|
* This method must run inside the global replication-range mutation lane.
|
|
15314
15718
|
*/
|
|
15315
15719
|
async deleteReplicationRangesCoherently(ranges, ownerHash, options) {
|
|
15720
|
+
const wasReplicator = this.uniqueReplicators.has(ownerHash);
|
|
15721
|
+
const joinWasEmitted = this._replicatorJoinEmitted.has(ownerHash);
|
|
15316
15722
|
if (ranges.length === 0) {
|
|
15317
15723
|
const ownerHasRanges = (await this.replicationIndex.count({ query: { hash: ownerHash } })) > 0;
|
|
15318
15724
|
if (!ownerHasRanges) {
|
|
@@ -15323,13 +15729,25 @@ let SharedLog = (() => {
|
|
|
15323
15729
|
removed: [],
|
|
15324
15730
|
retained: [],
|
|
15325
15731
|
ownerHasRanges,
|
|
15732
|
+
rollback: async () => {
|
|
15733
|
+
if (wasReplicator) {
|
|
15734
|
+
this.uniqueReplicators.add(ownerHash);
|
|
15735
|
+
}
|
|
15736
|
+
else {
|
|
15737
|
+
this.uniqueReplicators.delete(ownerHash);
|
|
15738
|
+
}
|
|
15739
|
+
if (joinWasEmitted) {
|
|
15740
|
+
this._replicatorJoinEmitted.add(ownerHash);
|
|
15741
|
+
}
|
|
15742
|
+
else {
|
|
15743
|
+
this._replicatorJoinEmitted.delete(ownerHash);
|
|
15744
|
+
}
|
|
15745
|
+
},
|
|
15326
15746
|
};
|
|
15327
15747
|
}
|
|
15328
15748
|
const uniqueRanges = [
|
|
15329
15749
|
...new Map(ranges.map((range) => [range.idString, range])).values(),
|
|
15330
15750
|
];
|
|
15331
|
-
const wasReplicator = this.uniqueReplicators.has(ownerHash);
|
|
15332
|
-
const joinWasEmitted = this._replicatorJoinEmitted.has(ownerHash);
|
|
15333
15751
|
const snapshots = uniqueRanges.map((range) => {
|
|
15334
15752
|
const pending = this.pendingMaturity.get(ownerHash)?.get(range.idString);
|
|
15335
15753
|
return {
|
|
@@ -15516,10 +15934,76 @@ let SharedLog = (() => {
|
|
|
15516
15934
|
outcomeError = new AggregateError(errors, "Replication-range deletion and post-state reconciliation failed");
|
|
15517
15935
|
this.poisonReplicationOwnership(outcomeError);
|
|
15518
15936
|
}
|
|
15937
|
+
let rollbackStarted = false;
|
|
15938
|
+
const rollback = async () => {
|
|
15939
|
+
if (rollbackStarted) {
|
|
15940
|
+
return;
|
|
15941
|
+
}
|
|
15942
|
+
rollbackStarted = true;
|
|
15943
|
+
const rollbackErrors = [];
|
|
15944
|
+
const removedIds = new Set(removed.map((range) => range.idString));
|
|
15945
|
+
for (const snapshot of snapshots) {
|
|
15946
|
+
if (!removedIds.has(snapshot.range.idString)) {
|
|
15947
|
+
continue;
|
|
15948
|
+
}
|
|
15949
|
+
try {
|
|
15950
|
+
await this.replicationIndex.put(snapshot.range);
|
|
15951
|
+
this.putNativeReplicationRange(snapshot.range);
|
|
15952
|
+
}
|
|
15953
|
+
catch (error) {
|
|
15954
|
+
rollbackErrors.push(error);
|
|
15955
|
+
}
|
|
15956
|
+
}
|
|
15957
|
+
for (const snapshot of snapshots) {
|
|
15958
|
+
if (!removedIds.has(snapshot.range.idString) ||
|
|
15959
|
+
!snapshot.pending ||
|
|
15960
|
+
this.pendingMaturity.get(ownerHash)?.has(snapshot.range.idString)) {
|
|
15961
|
+
continue;
|
|
15962
|
+
}
|
|
15963
|
+
try {
|
|
15964
|
+
this.schedulePendingMaturity(snapshot.pending.range, snapshot.pending.from, {
|
|
15965
|
+
rebalance: snapshot.pending.rebalance,
|
|
15966
|
+
waitMs: Math.max(0, snapshot.pending.expiresAt - Date.now()),
|
|
15967
|
+
}, snapshot.pending.ownershipLifecycleController);
|
|
15968
|
+
}
|
|
15969
|
+
catch (error) {
|
|
15970
|
+
rollbackErrors.push(error);
|
|
15971
|
+
}
|
|
15972
|
+
}
|
|
15973
|
+
try {
|
|
15974
|
+
if (wasReplicator) {
|
|
15975
|
+
this.uniqueReplicators.add(ownerHash);
|
|
15976
|
+
}
|
|
15977
|
+
else {
|
|
15978
|
+
this.uniqueReplicators.delete(ownerHash);
|
|
15979
|
+
}
|
|
15980
|
+
if (joinWasEmitted) {
|
|
15981
|
+
this._replicatorJoinEmitted.add(ownerHash);
|
|
15982
|
+
}
|
|
15983
|
+
else {
|
|
15984
|
+
this._replicatorJoinEmitted.delete(ownerHash);
|
|
15985
|
+
}
|
|
15986
|
+
}
|
|
15987
|
+
catch (error) {
|
|
15988
|
+
rollbackErrors.push(error);
|
|
15989
|
+
}
|
|
15990
|
+
try {
|
|
15991
|
+
await this.updateOldestTimestampFromIndex();
|
|
15992
|
+
}
|
|
15993
|
+
catch (error) {
|
|
15994
|
+
rollbackErrors.push(error);
|
|
15995
|
+
}
|
|
15996
|
+
if (rollbackErrors.length > 0) {
|
|
15997
|
+
const failure = new AggregateError(rollbackErrors, "Replication-range deletion rollback failed");
|
|
15998
|
+
this.poisonReplicationOwnership(failure);
|
|
15999
|
+
throw failure;
|
|
16000
|
+
}
|
|
16001
|
+
};
|
|
15519
16002
|
return {
|
|
15520
16003
|
removed,
|
|
15521
16004
|
retained,
|
|
15522
16005
|
ownerHasRanges,
|
|
16006
|
+
rollback,
|
|
15523
16007
|
error: outcomeError,
|
|
15524
16008
|
};
|
|
15525
16009
|
}
|
|
@@ -15532,7 +16016,8 @@ let SharedLog = (() => {
|
|
|
15532
16016
|
}
|
|
15533
16017
|
this._replicationInfoRequestByPeer.delete(peerHash);
|
|
15534
16018
|
}
|
|
15535
|
-
scheduleReplicationInfoRequests(peer, replicationLifecycleController = this._instanceLifecycle
|
|
16019
|
+
scheduleReplicationInfoRequests(peer, replicationLifecycleController = this._instanceLifecycle
|
|
16020
|
+
?.membershipLifecycleController) {
|
|
15536
16021
|
if (!replicationLifecycleController ||
|
|
15537
16022
|
!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
15538
16023
|
return;
|
|
@@ -15606,6 +16091,7 @@ let SharedLog = (() => {
|
|
|
15606
16091
|
// A destination stream is scoped to exactly one topic-subscription
|
|
15607
16092
|
// session. Abort the predecessor synchronously before either barrier can
|
|
15608
16093
|
// yield; a late queue completion must never enter the new session.
|
|
16094
|
+
this._v2Receive.clearPeer(peerHash);
|
|
15609
16095
|
this._v2Send.clearPeer(peerHash);
|
|
15610
16096
|
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
15611
16097
|
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
@@ -15720,58 +16206,41 @@ let SharedLog = (() => {
|
|
|
15720
16206
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
15721
16207
|
this._liveness.markReplicatorActivity(peerHash);
|
|
15722
16208
|
this._peerSessions.markOpen(peerHash, expectedSubscriptionEpoch);
|
|
15723
|
-
|
|
15724
|
-
//
|
|
15725
|
-
//
|
|
15726
|
-
//
|
|
15727
|
-
|
|
15728
|
-
|
|
15729
|
-
|
|
15730
|
-
|
|
15731
|
-
|
|
15732
|
-
|
|
15733
|
-
|
|
15734
|
-
capabilities: receiveCapabilities,
|
|
15735
|
-
}), {
|
|
15736
|
-
mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
|
|
16209
|
+
this.promoteReplicationInfoV2ReceiveCapability(publicKey, expectedSubscriptionEpoch);
|
|
16210
|
+
// Decode, sender and authenticated apply readiness are separate bits. An
|
|
16211
|
+
// ACKed capability advert is the local half of receiver-led negotiation;
|
|
16212
|
+
// readiness is promoted only after the legacy startup path has completed.
|
|
16213
|
+
// This preserves mixed-version discovery without putting capability ACK
|
|
16214
|
+
// latency on the subscription callback's critical path.
|
|
16215
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
16216
|
+
const localCapabilityAdvertisement = this._v2Receive.advertiseLocalCapability({
|
|
16217
|
+
target: publicKey,
|
|
16218
|
+
peerSession: expectedSubscriptionEpoch,
|
|
16219
|
+
receiveEpoch,
|
|
15737
16220
|
signal: replicationLifecycleController.signal,
|
|
15738
|
-
})
|
|
15739
|
-
.catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
|
|
15740
|
-
let replicationSegments;
|
|
16221
|
+
});
|
|
15741
16222
|
try {
|
|
15742
|
-
replicationSegments
|
|
15743
|
-
|
|
15744
|
-
|
|
15745
|
-
|
|
15746
|
-
|
|
15747
|
-
|
|
16223
|
+
let replicationSegments;
|
|
16224
|
+
try {
|
|
16225
|
+
replicationSegments = await this.getMyReplicationSegments();
|
|
16226
|
+
}
|
|
16227
|
+
catch (error) {
|
|
16228
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
16229
|
+
isNotStartedError(error)) {
|
|
16230
|
+
return;
|
|
16231
|
+
}
|
|
16232
|
+
throw error;
|
|
15748
16233
|
}
|
|
15749
|
-
throw error;
|
|
15750
|
-
}
|
|
15751
|
-
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
15752
|
-
!ownsSubscriptionEpoch()) {
|
|
15753
|
-
return;
|
|
15754
|
-
}
|
|
15755
|
-
if (replicationSegments.length > 0) {
|
|
15756
|
-
const segments = replicationSegments.map((x) => x.toReplicationRange());
|
|
15757
|
-
this.validatePersistedReplicationRangeSnapshot(segments);
|
|
15758
|
-
await this.rpc
|
|
15759
|
-
.send(new AllReplicatingSegmentsMessage({
|
|
15760
|
-
segments,
|
|
15761
|
-
}), {
|
|
15762
|
-
mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
|
|
15763
|
-
signal: replicationLifecycleController.signal,
|
|
15764
|
-
})
|
|
15765
|
-
.catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
|
|
15766
16234
|
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
15767
16235
|
!ownsSubscriptionEpoch()) {
|
|
15768
16236
|
return;
|
|
15769
16237
|
}
|
|
15770
|
-
if (
|
|
15771
|
-
|
|
16238
|
+
if (replicationSegments.length > 0) {
|
|
16239
|
+
const segments = replicationSegments.map((x) => x.toReplicationRange());
|
|
16240
|
+
this.validatePersistedReplicationRangeSnapshot(segments);
|
|
15772
16241
|
await this.rpc
|
|
15773
|
-
.send(new
|
|
15774
|
-
|
|
16242
|
+
.send(new AllReplicatingSegmentsMessage({
|
|
16243
|
+
segments,
|
|
15775
16244
|
}), {
|
|
15776
16245
|
mode: new AcknowledgeDelivery({
|
|
15777
16246
|
redundancy: 1,
|
|
@@ -15780,14 +16249,35 @@ let SharedLog = (() => {
|
|
|
15780
16249
|
signal: replicationLifecycleController.signal,
|
|
15781
16250
|
})
|
|
15782
16251
|
.catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
|
|
16252
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
16253
|
+
!ownsSubscriptionEpoch()) {
|
|
16254
|
+
return;
|
|
16255
|
+
}
|
|
16256
|
+
if (this.v8Behaviour) {
|
|
16257
|
+
// for backwards compatibility
|
|
16258
|
+
await this.rpc
|
|
16259
|
+
.send(new ResponseRoleMessage({
|
|
16260
|
+
role: this.getRoleFromReplicationSegments(replicationSegments),
|
|
16261
|
+
}), {
|
|
16262
|
+
mode: new AcknowledgeDelivery({
|
|
16263
|
+
redundancy: 1,
|
|
16264
|
+
to: [publicKey],
|
|
16265
|
+
}),
|
|
16266
|
+
signal: replicationLifecycleController.signal,
|
|
16267
|
+
})
|
|
16268
|
+
.catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
|
|
16269
|
+
}
|
|
16270
|
+
}
|
|
16271
|
+
// Keep legacy request-based discovery independent of the capability ACK.
|
|
16272
|
+
// This makes mixed-version joins resilient to timing-sensitive delivery/order
|
|
16273
|
+
// issues where we may miss the remote peer's initial announcement.
|
|
16274
|
+
if (this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
16275
|
+
ownsSubscriptionEpoch()) {
|
|
16276
|
+
this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
|
|
15783
16277
|
}
|
|
15784
16278
|
}
|
|
15785
|
-
|
|
15786
|
-
|
|
15787
|
-
// replication announcement.
|
|
15788
|
-
if (this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
15789
|
-
ownsSubscriptionEpoch()) {
|
|
15790
|
-
this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
|
|
16279
|
+
finally {
|
|
16280
|
+
localCapabilityAdvertisement.releaseLegacyBarrier();
|
|
15791
16281
|
}
|
|
15792
16282
|
}
|
|
15793
16283
|
getClampedReplicas(customValue) {
|