@peerbit/shared-log 13.2.30 → 13.2.32

package/dist/src/index.js

@@ -54,7 +54,7 @@ import { CheckedPruneCoordinator, } from "./checked-prune.js";
 import { CPUUsageIntervalLag } from "./cpu.js";
 import { debouncedAccumulatorMap, } from "./debounce.js";
 import { NativeDurableCommitError, NoPeersError, isNotStartedError, } from "./errors.js";
-import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
+import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
 import { FanoutEnvelope } from "./fanout-envelope.js";
 import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isPromiseLike, mapMaybePromise, normalizedHashValues, } from "./coordinate-persistence.js";
 import { InstanceLifecycle } from "./instance-lifecycle.js";
@@ -67,10 +67,11 @@ import { PeerSessionRegistry, } from "./peer-session.js";
 import { PIDReplicationController } from "./pid.js";
 import { EntryReplicatedU32, EntryReplicatedU64, ReplicationIntent, ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, ReplicationRangeMessage, appromixateCoverage, calculateCoverage, countCoveringRangesSameOwner, createAssignedRangesQuery, debounceAggregationChanges, getAllMergeCandiates, getCoverSet, getSamples, isEntryReplicated, isMatured, isReplicationRangeMessage, mergeRanges, minimumWidthToCover, shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary, toRebalance, } from "./ranges.js";
 import { ReplicationAnnouncementCoordinator, isTransientReplicationAnnouncementError, } from "./replication-announcement.js";
+import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
 import { createReplicationDomainHash, } from "./replication-domain-hash.js";
 import { createReplicationDomainTime, } from "./replication-domain-time.js";
 import {} from "./replication-domain.js";
-import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, MinReplicas, ReplicationError, ReplicationPingMessage, RequestReplicationInfoMessage, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
+import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, MinReplicas, ReplicationError, ReplicationPingMessage, RequestReplicationInfoMessage, RequestReplicationInfoV2Message, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, isReplicationInfoV2Message, maxReplicas, } from "./replication.js";
 import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
 import { Observer, Replicator } from "./role.js";
 import { createSyncronizer } from "./sync/factory.js";
@@ -1608,6 +1609,7 @@ let SharedLog = (() => {
         // A fn that we can call many times that recalculates the participation role
         rebalanceParticipationDebounced;
         _announcements;
+        _v2Send;
         // A fn for debouncing the calls for pruning
         pruneDebouncedFn;
         _checkedPruneAuditTimer;
@@ -1637,6 +1639,10 @@ let SharedLog = (() => {
         // Sync capability bits advertised by peers (SyncCapabilitiesMessage), keyed
         // by public key hash. Entries are dropped on unsubscribe/disconnect.
         _peerSyncCapabilities;
+        // Signed transport session carried by the capability envelope. Kept in a
+        // parallel map so existing capability-number consumers remain unchanged.
+        _peerSyncCapabilitySessions;
+        _peerSyncCapabilityTimestamps;
         // Pending live raw exchange-head gossip, coalesced per recipient set and
         // flushed at the end of the current event-loop turn (or when a batch cap
         // is hit). Only used when every recipient advertised raw capability.
@@ -1690,6 +1696,7 @@ let SharedLog = (() => {
                 // this).
                 queueCurrentReplicationStateAnnouncementRepair: () => this._announcements.queueCurrentReplicationStateAnnouncementRepair(),
                 queueCurrentReplicationStateAnnouncementRetry: (error) => this._announcements.queueCurrentReplicationStateAnnouncementRetry(error),
+                enqueueReplicationInfoV2: (message) => this._v2Send.enqueue(message),
                 isClosed: () => this.closed,
                 getCloseSignal: () => this._closeController.signal,
                 getMyReplicationSegments: () => this.getMyReplicationSegments(),
@@ -1704,6 +1711,22 @@ let SharedLog = (() => {
                 callRebalanceParticipationDebounced: () => this.rebalanceParticipationDebounced?.call(),
             });
         }
+        createReplicationInfoV2SendCoordinator() {
+            return new ReplicationInfoV2SendCoordinator({
+                getRpc: () => this.rpc,
+                getSelfKey: () => this.node.identity.publicKey,
+                getSenderTransportSession: () => BigInt(this.node.services.pubsub.session),
+                getMyReplicationSegments: () => this.getMyReplicationSegments(),
+                validatePersistedReplicationRangeSnapshot: (ranges) => this.validatePersistedReplicationRangeSnapshot(ranges),
+                isClosed: () => this.closed,
+                isPeerSessionOpen: (peerHash, peerSession) => this._peerSessions.isCurrent(peerHash, peerSession) &&
+                    peerSession.phase === "open" &&
+                    !this._peerSessions.isReplicationInfoBlocked(peerHash) &&
+                    this._peerSessions.isReceiveCleanupGateOpen(peerHash),
+                captureReplicationOwnershipLifecycle: () => this.captureReplicationOwnershipLifecycle(),
+                isReplicationOwnershipLifecycleActive: (controller) => this.isRepairLifecycleActive(controller),
+            });
+        }
         createReplicatorLivenessMonitor() {
             return new ReplicatorLivenessMonitor({
                 // Sweep-driven probes and activity marks dispatch through the owner so
@@ -1798,6 +1821,8 @@ let SharedLog = (() => {
             this._appendBackfillPendingByTarget = new Map();
             this._topicSubscribersCache = new Map();
             this._peerSyncCapabilities = new Map();
+            this._peerSyncCapabilitySessions = new Map();
+            this._peerSyncCapabilityTimestamps = new Map();
             this._liveRawGossipBatches = new Map();
             this._liveRawGossipFlushScheduled = false;
             this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
@@ -1806,6 +1831,7 @@ let SharedLog = (() => {
             this._replicatorJoinEmitted = new Set();
             this._replicatorsReconciled = false;
             this._liveness = this.createReplicatorLivenessMonitor();
+            this._v2Send = this.createReplicationInfoV2SendCoordinator();
             this._announcements = this.createReplicationAnnouncementCoordinator();
             this.pendingMaturity = new Map();
             this._closeController = new AbortController();
@@ -2196,6 +2222,72 @@ let SharedLog = (() => {
                 SYNC_CAPABILITY_RAW_EXCHANGE_HEADS) !==
                 0);
         }
+        observePeerSyncCapabilities(properties) {
+            const { peerHash, capabilities, transportSession, timestamp, openingSession, } = properties;
+            if (openingSession) {
+                const previous = this._openingSyncCapabilitiesByPeer.get(peerHash);
+                if (transportSession !== undefined &&
+                    timestamp !== undefined &&
+                    previous?.epoch === openingSession &&
+                    previous.transportSession === transportSession) {
+                    if (previous.timestamp !== undefined && timestamp < previous.timestamp) {
+                        return false;
+                    }
+                    this._openingSyncCapabilitiesByPeer.set(peerHash, {
+                        epoch: openingSession,
+                        capabilities: previous.capabilities | capabilities,
+                        transportSession,
+                        timestamp: previous.timestamp === undefined || timestamp > previous.timestamp
+                            ? timestamp
+                            : previous.timestamp,
+                    });
+                    return true;
+                }
+                this._openingSyncCapabilitiesByPeer.set(peerHash, {
+                    epoch: openingSession,
+                    capabilities,
+                    transportSession,
+                    timestamp,
+                });
+                return true;
+            }
+            if (transportSession === undefined || timestamp === undefined) {
+                // Test/in-process synthetic contexts predate signed envelope captures.
+                // They may exercise capability-number behavior, but can never authorize V2.
+                this._peerSyncCapabilities.set(peerHash, capabilities);
+                this._peerSyncCapabilitySessions.delete(peerHash);
+                this._peerSyncCapabilityTimestamps.delete(peerHash);
+                this._v2Send.advancePeerCapability(peerHash);
+                return true;
+            }
+            const previousSession = this._peerSyncCapabilitySessions.get(peerHash);
+            const previousTimestamp = this._peerSyncCapabilityTimestamps.get(peerHash);
+            const sameTransportSession = previousSession === transportSession;
+            if (sameTransportSession &&
+                previousTimestamp !== undefined &&
+                timestamp < previousTimestamp) {
+                return false;
+            }
+            const previousCapabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
+            const nextCapabilities = sameTransportSession
+                ? previousCapabilities | capabilities
+                : capabilities;
+            const generationAdvanced = !sameTransportSession ||
+                previousTimestamp === undefined ||
+                timestamp > previousTimestamp ||
+                nextCapabilities !== previousCapabilities;
+            this._peerSyncCapabilities.set(peerHash, nextCapabilities);
+            this._peerSyncCapabilitySessions.set(peerHash, transportSession);
+            this._peerSyncCapabilityTimestamps.set(peerHash, previousTimestamp === undefined ||
+                !sameTransportSession ||
+                timestamp > previousTimestamp
+                ? timestamp
+                : previousTimestamp);
+            if (generationAdvanced) {
+                this._v2Send.advancePeerCapability(peerHash);
+            }
+            return true;
+        }
         /**
          * Live append gossip may use the raw exchange-heads path only when we
          * opted into raw sync and every remote recipient advertised raw capability
@@ -8895,6 +8987,8 @@ let SharedLog = (() => {
                 ttl: LEADER_PLAN_CACHE_TTL_MS,
             });
             this._peerSyncCapabilities = new Map();
+            this._peerSyncCapabilitySessions = new Map();
+            this._peerSyncCapabilityTimestamps = new Map();
             this._liveRawGossipBatches = new Map();
             this._liveRawGossipFlushScheduled = false;
             this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
@@ -8910,6 +9004,8 @@ let SharedLog = (() => {
             this._lastLocalAppendAt = 0;
             this._announcements ??= this.createReplicationAnnouncementCoordinator();
             this._announcements.resetForOpen();
+            this._v2Send ??= this.createReplicationInfoV2SendCoordinator();
+            this._v2Send.resetForOpen();
             const adaptiveReplicateOptions = options?.replicate && isAdaptiveReplicatorOption(options.replicate)
                 ? options.replicate
                 : undefined;
@@ -10029,6 +10125,9 @@ let SharedLog = (() => {
             this._liveness._replicatorLivenessFailures.delete(peerHash);
             this._liveness._replicatorLastActivityAt.delete(peerHash);
             this._peerSyncCapabilities.delete(peerHash);
+            this._peerSyncCapabilitySessions.delete(peerHash);
+            this._peerSyncCapabilityTimestamps.delete(peerHash);
+            this._v2Send.clearPeer(peerHash);
             this.cleanupPendingIHavePeer(peerHash);
             this.cleanupCheckedPrunePeer(peerHash, ownershipLifecycleController, this._checkedPrune);
         }
@@ -10845,6 +10944,9 @@ let SharedLog = (() => {
                 this._openingSyncCapabilitiesByPeer?.clear();
                 this._gidPeersHistory?.clear();
                 this._peerSyncCapabilities?.clear();
+                this._peerSyncCapabilitySessions?.clear();
+                this._peerSyncCapabilityTimestamps?.clear();
+                this._v2Send?.clearForClose();
                 this._liveRawGossipBatches?.clear();
                 this._nativeSharedLogState?.clearGidPeers();
                 this._nativeBackbone?.clearGidPeers();
@@ -10965,12 +11067,14 @@ let SharedLog = (() => {
                         }
                     }, 2_000);
                     try {
-                        await this.rpc
-                            .send(new AllReplicatingSegmentsMessage({ segments: [] }), {
-                            priority: CONVERGENCE_MESSAGE_PRIORITY,
-                            signal: abort.signal,
-                        })
-                            .catch(() => { });
+                        const reset = new AllReplicatingSegmentsMessage({ segments: [] });
+                        await Promise.all([
+                            this.rpc.send(reset, {
+                                priority: CONVERGENCE_MESSAGE_PRIORITY,
+                                signal: abort.signal,
+                            }).catch(() => { }),
+                            this._v2Send.sendTerminalReset(abort.signal),
+                        ]);
                     }
                     finally {
                         clearTimeout(abortTimer);
@@ -11088,12 +11192,14 @@ let SharedLog = (() => {
                         }
                     }, 2_000);
                     try {
-                        await this.rpc
-                            .send(new AllReplicatingSegmentsMessage({ segments: [] }), {
-                            priority: CONVERGENCE_MESSAGE_PRIORITY,
-                            signal: abort.signal,
-                        })
-                            .catch(() => { });
+                        const reset = new AllReplicatingSegmentsMessage({ segments: [] });
+                        await Promise.all([
+                            this.rpc.send(reset, {
+                                priority: CONVERGENCE_MESSAGE_PRIORITY,
+                                signal: abort.signal,
+                            }).catch(() => { }),
+                            this._v2Send.sendTerminalReset(abort.signal),
+                        ]);
                     }
                     finally {
                         clearTimeout(abortTimer);
@@ -11508,6 +11614,13 @@ let SharedLog = (() => {
         }
         // Callback for receiving a message from the network
         async onMessage(msg, context) {
+            // Decode-first only: capability advertisement is not permission to use
+            // V2. Drop every unsolicited V2 envelope before even entering the shared
+            // receive envelope, so durable poison checks, leases, liveness, ordering
+            // watermarks, replication ranges and cleanup side effects cannot observe it.
+            if (isReplicationInfoV2Message(msg)) {
+                return;
+            }
             const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
                 ? msg
                 : undefined;
@@ -11558,7 +11671,8 @@ let SharedLog = (() => {
                     // allocation cost, and before liveness or apply-queue side effects.
                     this.validateStoppedReplicationAnnouncement(msg.segmentIds);
                 }
-                if (!context.from.equals(this.node.identity.publicKey)) {
+                if (!context.from.equals(this.node.identity.publicKey) &&
+                    !(msg instanceof RequestReplicationInfoV2Message)) {
                     this._liveness.markReplicatorActivity(receiveFromHash);
                 }
                 const syncProfile = this._logProperties?.sync?.profile;
@@ -12901,6 +13015,8 @@ let SharedLog = (() => {
                     }
                     else if (msg instanceof SyncCapabilitiesMessage) {
                         if (!context.from.equals(this.node.identity.publicKey)) {
+                            const capabilityTransportSession = context.message?.header?.session;
+                            const capabilityTimestamp = context.message?.header?.timestamp;
                             // No await separates this from the capture above, so the captured
                             // window state is exact: the legacy re-read of the opening map here
                             // could never observe a different value.
@@ -12909,17 +13025,47 @@ let SharedLog = (() => {
                                 // A prior unsubscribe cleanup may still be ahead of this reconnect
                                 // barrier. Stage the new generation's one-shot advertisement so that
                                 // cleanup cannot erase it before the opening transition commits.
-                                this._openingSyncCapabilitiesByPeer.set(receiveFromHash, {
-                                    epoch: receiveSession,
+                                this.observePeerSyncCapabilities({
+                                    peerHash: receiveFromHash,
                                     capabilities: msg.capabilities,
+                                    transportSession: capabilityTransportSession,
+                                    timestamp: capabilityTimestamp,
+                                    openingSession: receiveSession,
                                 });
                             }
                             else {
-                                this._peerSyncCapabilities.set(receiveFromHash, msg.capabilities);
+                                this.observePeerSyncCapabilities({
+                                    peerHash: receiveFromHash,
+                                    capabilities: msg.capabilities,
+                                    transportSession: capabilityTransportSession,
+                                    timestamp: capabilityTimestamp,
+                                });
                             }
                         }
                         return;
                     }
+                    else if (msg instanceof RequestReplicationInfoV2Message) {
+                        const requiredCapabilities = SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
+                            SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY;
+                        if (receiveSession &&
+                            ((this._peerSyncCapabilities.get(receiveFromHash) ?? 0) &
+                                requiredCapabilities) ===
+                                requiredCapabilities &&
+                            this._peerSyncCapabilitySessions.get(receiveFromHash) ===
+                                context.message.header.session &&
+                            this._peerSyncCapabilityTimestamps.has(receiveFromHash) &&
+                            context.message.header.timestamp >
+                                this._peerSyncCapabilityTimestamps.get(receiveFromHash) &&
+                            this._v2Send.acceptRequest(msg, {
+                                from: context.from,
+                                peerSession: receiveSession,
+                                receiverTransportSession: context.message.header.session,
+                                requestTimestamp: context.message.header.timestamp,
+                            })) {
+                            this._liveness.markReplicatorActivity(receiveFromHash);
+                        }
+                        return;
+                    }
                     else if (await this.syncronizer.onMessage(msg, context)) {
                         return; // the syncronizer has handled the message
                     }
@@ -13155,6 +13301,7 @@ let SharedLog = (() => {
             }
             const segments = replicationSegments.map((x) => x.toReplicationRange());
             this.validatePersistedReplicationRangeSnapshot(segments);
+            this._v2Send.enqueueSnapshotForPeer(receiveFromHash);
             await this.rpc
                 .send(new AllReplicatingSegmentsMessage({ segments }), {
                 mode: new AcknowledgeDelivery({
@@ -15456,6 +15603,12 @@ let SharedLog = (() => {
             if (!ownsSubscriptionEpoch()) {
                 return;
             }
+            // A destination stream is scoped to exactly one topic-subscription
+            // session. Abort the predecessor synchronously before either barrier can
+            // yield; a late queue completion must never enter the new session.
+            this._v2Send.clearPeer(peerHash);
+            this._peerSyncCapabilitySessions.delete(peerHash);
+            this._peerSyncCapabilityTimestamps.delete(peerHash);
             if (subscribed) {
                 const pendingOpeningCapabilities = this._openingSyncCapabilitiesByPeer.get(peerHash);
                 if (pendingOpeningCapabilities &&
@@ -15486,6 +15639,18 @@ let SharedLog = (() => {
                     const openingCapabilities = this._openingSyncCapabilitiesByPeer.get(peerHash);
                     if (openingCapabilities?.epoch === expectedSubscriptionEpoch) {
                         this._peerSyncCapabilities.set(peerHash, openingCapabilities.capabilities);
+                        if (openingCapabilities.transportSession === undefined) {
+                            this._peerSyncCapabilitySessions.delete(peerHash);
+                        }
+                        else {
+                            this._peerSyncCapabilitySessions.set(peerHash, openingCapabilities.transportSession);
+                        }
+                        if (openingCapabilities.timestamp === undefined) {
+                            this._peerSyncCapabilityTimestamps.delete(peerHash);
+                        }
+                        else {
+                            this._peerSyncCapabilityTimestamps.set(peerHash, openingCapabilities.timestamp);
+                        }
                         this._openingSyncCapabilitiesByPeer.delete(peerHash);
                     }
                     this._peerSessions.unblockReplicationInfo(peerHash);
@@ -15555,19 +15720,23 @@ let SharedLog = (() => {
             this._liveness._replicatorLivenessFailures.delete(peerHash);
             this._liveness.markReplicatorActivity(peerHash);
             this._peerSessions.markOpen(peerHash, expectedSubscriptionEpoch);
-            if (this._logProperties?.sync?.rawExchangeHeads === true) {
-                // One-shot capability advertisement so live append gossip can pick
-                // the raw exchange-heads path for this peer without a per-request
-                // round trip. Peers that do not know the message drop it.
-                this.rpc
-                    .send(new SyncCapabilitiesMessage({
-                    capabilities: SYNC_CAPABILITY_RAW_EXCHANGE_HEADS,
-                }), {
-                    mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
-                    signal: replicationLifecycleController.signal,
-                })
-                    .catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
-            }
+            // Decode support and receiver-led negotiation readiness are separate bits.
+            // This node never initiates a request in this rollout; normal replication
+            // announcements remain legacy unless a peer explicitly grants one signed,
+            // session-bound V2 destination stream.
+            const receiveCapabilities = SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
+                SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
+                (this._logProperties?.sync?.rawExchangeHeads === true
+                    ? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
+                    : 0);
+            this.rpc
+                .send(new SyncCapabilitiesMessage({
+                capabilities: receiveCapabilities,
+            }), {
+                mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
+                signal: replicationLifecycleController.signal,
+            })
+                .catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
             let replicationSegments;
             try {
                 replicationSegments = await this.getMyReplicationSegments();