@peerbit/shared-log 16.0.24 → 16.0.26

package/dist/src/index.js

@@ -55,7 +55,7 @@ import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isProm
 import { CPUUsageIntervalLag } from "./cpu.js";
 import { debouncedAccumulatorMap, } from "./debounce.js";
 import { CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, PersistedDeliveryError, isNotStartedError, } from "./errors.js";
-import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
+import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_REARM, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
 import { FanoutEnvelope } from "./fanout-envelope.js";
 import { InstanceLifecycle } from "./instance-lifecycle.js";
 import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
@@ -2024,12 +2024,20 @@ let SharedLog = (() => {
             const peerHash = properties.target.hashcode();
             const receiverTransportSession = this.ownTransportSession();
             await this.rpc.send(new SyncCapabilitiesMessage({
-                capabilities: this.replicationInfoV2ReceiveCapabilities(),
+                capabilities: this.replicationInfoV2ReceiveCapabilities() |
+                    (properties.requestRemoteFullRearm
+                        ? SYNC_CAPABILITY_REPLICATION_INFO_V2_REARM
+                        : 0),
             }), {
-                mode: new AcknowledgeDelivery({
-                    redundancy: 1,
-                    to: [properties.target],
-                }),
+                // The transient rearm is only a bounded recovery hint. The ensuing
+                // authenticated Full/Applied exchange is the readiness proof, so do not
+                // let an ACK-delivery promise park the recovery scheduler itself.
+                mode: properties.requestRemoteFullRearm
+                    ? new SilentDelivery({ redundancy: 1, to: [properties.target] })
+                    : new AcknowledgeDelivery({
+                        redundancy: 1,
+                        to: [properties.target],
+                    }),
                 priority: CONVERGENCE_MESSAGE_PRIORITY,
                 signal: properties.signal,
             });
@@ -2078,6 +2086,7 @@ let SharedLog = (() => {
                     peerSession: properties.peerSession,
                     receiveEpoch: properties.receiveEpoch,
                     signal: properties.signal,
+                    requestRemoteFullRearm: properties.requestRemoteFullRearm,
                 }),
                 onRequestError: (error) => {
                     if (isNotStartedError(error) ||
@@ -3447,7 +3456,7 @@ let SharedLog = (() => {
                 })) {
                 return undefined;
             }
-            return { capabilitySession, peerSession };
+            return { capabilitySession, peerSession, receiveEpoch };
         }
         async waitPersistedReceiptRetry(signal, ms) {
             try {
@@ -15966,6 +15975,13 @@ let SharedLog = (() => {
                         if (!context.from.equals(this.node.identity.publicKey)) {
                             const capabilityTransportSession = context.message?.header?.session;
                             const capabilityTimestamp = context.message?.header?.timestamp;
+                            const previousCapabilitySession = this._peerSyncCapabilitySessions.get(receiveFromHash);
+                            const previousCapabilityTimestamp = this._peerSyncCapabilityTimestamps.get(receiveFromHash);
+                            const requestsRemoteFullRearm = (msg.capabilities & SYNC_CAPABILITY_REPLICATION_INFO_V2_REARM) !==
+                                0;
+                            // The rearm bit is a one-shot authenticated command, not a sticky
+                            // negotiated capability. Store and promote only the steady bits.
+                            const steadyCapabilities = msg.capabilities & ~SYNC_CAPABILITY_REPLICATION_INFO_V2_REARM;
                             // No await separates this from the capture above, so the captured
                             // window state is exact: the legacy re-read of the opening map here
                             // could never observe a different value.
@@ -15976,7 +15992,7 @@ let SharedLog = (() => {
                                 // cleanup cannot erase it before the opening transition commits.
                                 this.observePeerSyncCapabilities({
                                     peerHash: receiveFromHash,
-                                    capabilities: msg.capabilities,
+                                    capabilities: steadyCapabilities,
                                     transportSession: capabilityTransportSession,
                                     timestamp: capabilityTimestamp,
                                     openingSession: receiveSession,
@@ -15985,12 +16001,34 @@ let SharedLog = (() => {
                             else {
                                 const observed = this.observePeerSyncCapabilities({
                                     peerHash: receiveFromHash,
-                                    capabilities: msg.capabilities,
+                                    capabilities: steadyCapabilities,
                                     transportSession: capabilityTransportSession,
                                     timestamp: capabilityTimestamp,
                                 });
                                 if (observed && receiveSession?.phase === "open") {
                                     this.promoteReplicationInfoV2ReceiveCapability(context.from, receiveSession);
+                                    const freshExactRearm = requestsRemoteFullRearm &&
+                                        capabilityTransportSession !== undefined &&
+                                        capabilityTimestamp !== undefined &&
+                                        previousCapabilitySession === capabilityTransportSession &&
+                                        previousCapabilityTimestamp !== undefined &&
+                                        capabilityTimestamp > previousCapabilityTimestamp;
+                                    if (freshExactRearm &&
+                                        this._v2Receive.isCurrentActive({
+                                            peerHash: receiveFromHash,
+                                            peerSession: receiveSession,
+                                            receiveEpoch: this._peerSessions.receiveEpoch(receiveFromHash),
+                                            senderTransportSession: capabilityTransportSession,
+                                        })) {
+                                        // Rotate the receiver grant/challenge before requesting Full. A
+                                        // sender rebuilt from no state starts at sequence one, which an
+                                        // active receiver's old sequence fence must otherwise reject.
+                                        this._v2Receive.advanceRecovery({
+                                            peerHash: receiveFromHash,
+                                            peerSession: receiveSession,
+                                            receiveEpoch: this._peerSessions.receiveEpoch(receiveFromHash),
+                                        });
+                                    }
                                 }
                                 else if (observed && receiveSession === null) {
                                     // A capability can arrive before the sender's topic Subscribe after
@@ -16855,8 +16893,8 @@ let SharedLog = (() => {
             }
             throwIfInactive();
         }
-        nudgePersistedReceiptPeerReadiness(publicKey) {
-            if (this.closed)
+        nudgePersistedReceiptPeerReadiness(publicKey, signal, rearmCurrentUnconfirmed) {
+            if (this.closed || signal.aborted)
                 return;
             const peerHash = publicKey.hashcode();
             const peerSession = this._peerSessions.current(peerHash);
@@ -16886,6 +16924,29 @@ let SharedLog = (() => {
                 peerSession,
                 receiveEpoch,
             });
+            const receiptTarget = this.persistedReceiptPeerSession(peerHash);
+            if (receiptTarget &&
+                (!this._v2Send.hasCurrentStateForPeer({
+                    peerHash,
+                    peerSession: receiptTarget.peerSession,
+                    receiverTransportSession: receiptTarget.capabilitySession,
+                }) ||
+                    (rearmCurrentUnconfirmed?.peerSession === receiptTarget.peerSession &&
+                        rearmCurrentUnconfirmed.receiveEpoch === receiptTarget.receiveEpoch &&
+                        rearmCurrentUnconfirmed.capabilitySession ===
+                            receiptTarget.capabilitySession &&
+                        !this._v2Send.isLatestConfirmedForPeer({
+                            peerHash,
+                            peerSession: receiptTarget.peerSession,
+                            receiverTransportSession: receiptTarget.capabilitySession,
+                        })))) {
+                this._v2Receive.reAdvertiseLocalCapabilityForRemoteFull({
+                    peerHash,
+                    peerSession: receiptTarget.peerSession,
+                    receiveEpoch: receiptTarget.receiveEpoch,
+                    signal,
+                });
+            }
             this.scheduleReplicationInfoV2Recovery(publicKey);
         }
         /**
@@ -16973,7 +17034,9 @@ let SharedLog = (() => {
          * Wait for a public key's current (or replacement) connection generation to
          * become persisted-receipt ready. Transition listeners are installed before
          * the first asynchronous inspection, and a bounded recovery tick repairs
-         * missed subscriber/capability wakes without retaining stale PeerSessions.
+         * missed subscriber/capability wakes without retaining stale PeerSessions. A
+         * current sender gets one normal confirmation interval before an exact-
+         * generation watchdog requests a bounded Full rearm.
          * This waiter is advisory only; the following persisted delivery remains the
          * operation that proves the requested remote durability quorum.
          */
@@ -17017,6 +17080,7 @@ let SharedLog = (() => {
             let rerun = false;
             let recoveryTimer;
             let confirmationController;
+            let confirmationRecoveryTarget;
             let lastSnapshot;
             const createTimeoutError = () => {
                 const suffix = lastSnapshot
@@ -17037,6 +17101,7 @@ let SharedLog = (() => {
                 }
                 confirmationController?.abort(new AbortError("Persisted-receipt readiness generation changed"));
                 confirmationController = undefined;
+                confirmationRecoveryTarget = undefined;
                 operationController.abort(new AbortError("Persisted-receipt readiness wait settled"));
             };
             const resolve = (snapshot) => {
@@ -17079,19 +17144,52 @@ let SharedLog = (() => {
                     throw new AbortError("Persisted-receipt readiness wait settled");
                 }
             };
+            const recoveryDelayMs = Math.max(50, Math.min(1_000, this.waitForReplicatorRequestIntervalMs));
             const armRecoveryTick = () => {
                 if (settled || recoveryTimer)
                     return;
-                const delayMs = Math.max(50, Math.min(1_000, this.waitForReplicatorRequestIntervalMs));
                 recoveryTimer = setTimeout(() => {
                     recoveryTimer = undefined;
                     if (!continueWait())
                         return;
-                    this.nudgePersistedReceiptPeerReadiness(key);
+                    const recoveryTarget = confirmationRecoveryTarget;
+                    this.nudgePersistedReceiptPeerReadiness(key, operationSignal, recoveryTarget && Date.now() >= recoveryTarget.notBefore
+                        ? recoveryTarget
+                        : undefined);
+                    if (checkInFlight) {
+                        // Confirmation can legitimately span several recovery intervals.
+                        // Keep repairing the exact current generation without aborting its
+                        // one application-confirmation waiter on every tick.
+                        rerun = true;
+                        armRecoveryTick();
+                        return;
+                    }
                     scheduleCheck();
-                }, delayMs);
+                }, recoveryDelayMs);
                 recoveryTimer.unref?.();
             };
+            const prepareConfirmationRecovery = () => {
+                const target = this.persistedReceiptPeerSession(peerHash);
+                if (!target) {
+                    confirmationRecoveryTarget = undefined;
+                    return undefined;
+                }
+                const retained = confirmationRecoveryTarget;
+                if (retained?.peerSession !== target.peerSession ||
+                    retained.receiveEpoch !== target.receiveEpoch ||
+                    retained.capabilitySession !== target.capabilitySession) {
+                    confirmationRecoveryTarget = {
+                        peerSession: target.peerSession,
+                        receiveEpoch: target.receiveEpoch,
+                        capabilitySession: target.capabilitySession,
+                        notBefore: Date.now() + recoveryDelayMs,
+                    };
+                }
+                const recoveryTarget = confirmationRecoveryTarget;
+                this.nudgePersistedReceiptPeerReadiness(key, operationSignal, Date.now() >= recoveryTarget.notBefore ? recoveryTarget : undefined);
+                armRecoveryTick();
+                return target;
+            };
             const runCheck = async () => {
                 checkScheduled = false;
                 if (!continueWait())
@@ -17106,9 +17204,10 @@ let SharedLog = (() => {
                     lastSnapshot = snapshot;
                     if (!continueWait())
                         return;
-                    if (rerun)
-                        return;
                     if (snapshot.status === "ready") {
+                        confirmationRecoveryTarget = undefined;
+                        if (rerun)
+                            return;
                         // A wake observed while the asynchronous inspection was running may
                         // already have invalidated this snapshot. Drain that coalesced wake
                         // before publishing readiness.
@@ -17117,10 +17216,17 @@ let SharedLog = (() => {
                     }
                     if (snapshot.status === "pending" &&
                         snapshot.reason === "replication-confirmation-pending") {
-                        const target = this.persistedReceiptPeerSession(peerHash);
+                        // Retain one watchdog deadline for the exact generation across
+                        // coalesced inspection wakes. A sub-interval event stream must drive,
+                        // rather than indefinitely postpone, the mature recovery nudge.
+                        const target = prepareConfirmationRecovery();
+                        if (rerun)
+                            return;
                         if (target) {
                             const currentConfirmationController = new AbortController();
                             confirmationController = currentConfirmationController;
+                            const currentConfirmationRecoveryTarget = confirmationRecoveryTarget;
+                            let confirmationSucceeded = false;
                             try {
                                 await this._v2Send.confirmLatestForPeer({
                                     peerHash,
@@ -17133,6 +17239,7 @@ let SharedLog = (() => {
                                         currentConfirmationController.signal,
                                     ]),
                                 });
+                                confirmationSucceeded = true;
                             }
                             catch (error) {
                                 if (!continueWait())
@@ -17146,6 +17253,10 @@ let SharedLog = (() => {
                                 if (confirmationController === currentConfirmationController) {
                                     confirmationController = undefined;
                                 }
+                                if (confirmationSucceeded &&
+                                    confirmationRecoveryTarget === currentConfirmationRecoveryTarget) {
+                                    confirmationRecoveryTarget = undefined;
+                                }
                             }
                             if (!continueWait())
                                 return;
@@ -17153,17 +17264,32 @@ let SharedLog = (() => {
                             lastSnapshot = snapshot;
                             if (!continueWait())
                                 return;
-                            if (rerun)
-                                return;
                             if (snapshot.status === "ready") {
+                                confirmationRecoveryTarget = undefined;
+                                if (rerun)
+                                    return;
                                 resolve(snapshot);
                                 return;
                             }
+                            if (snapshot.status === "pending" &&
+                                snapshot.reason === "replication-confirmation-pending") {
+                                prepareConfirmationRecovery();
+                            }
+                            else {
+                                confirmationRecoveryTarget = undefined;
+                            }
+                            if (rerun)
+                                return;
                         }
                     }
+                    else {
+                        confirmationRecoveryTarget = undefined;
+                    }
+                    if (rerun)
+                        return;
                     if (!continueWait())
                         return;
-                    this.nudgePersistedReceiptPeerReadiness(key);
+                    this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
                 }
                 catch (error) {
                     if (!settled)