@peerbit/shared-log 16.0.12 → 16.0.14

package/dist/src/index.js

@@ -55,7 +55,7 @@ import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isProm
 import { CPUUsageIntervalLag } from "./cpu.js";
 import { debouncedAccumulatorMap, } from "./debounce.js";
 import { CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, isNotStartedError, } from "./errors.js";
-import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
+import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
 import { FanoutEnvelope } from "./fanout-envelope.js";
 import { InstanceLifecycle } from "./instance-lifecycle.js";
 import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
@@ -66,14 +66,15 @@ import { NativeBackboneWriteThroughBlockStore } from "./native-write-through-blo
 import { PeerSessionRegistry } from "./peer-session.js";
 import { PIDReplicationController } from "./pid.js";
 import { EntryReplicatedU32, EntryReplicatedU64, ReplicationIntent, ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, ReplicationRangeMessage, appromixateCoverage, calculateCoverage, countCoveringRangesSameOwner, createAssignedRangesQuery, debounceAggregationChanges, getAllMergeCandiates, getCoverSet, getSamples, isEntryReplicated, isMatured, isReplicationRangeMessage, mergeRanges, minimumWidthToCover, shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary, toRebalance, } from "./ranges.js";
+import { receiveSignatureVerificationResult as getReceiveSignatureVerificationResult, withReceiveSignatureVerificationFacts as withSignatureVerificationFacts, } from "./receive-signature-verification.js";
 import { ReplicationAnnouncementCoordinator } from "./replication-announcement.js";
 import { createReplicationDomainHash, } from "./replication-domain-hash.js";
 import { createReplicationDomainTime, } from "./replication-domain-time.js";
 import {} from "./replication-domain.js";
 import { ReplicationInfoV2ReceiveCoordinator } from "./replication-info-v2-receive.js";
-import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
+import { ReplicationInfoV2SendCoordinator, } from "./replication-info-v2-send.js";
 import { classifyReplicationStatus, } from "./replication-status.js";
-import { AbsoluteReplicas, AddedReplicationInfoV2Message, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, FullReplicationInfoV2Message, MinReplicas, ReplicationError, ReplicationPingMessage, RequestReplicationInfoMessage, RequestReplicationInfoV2Message, ResponseRoleMessage, StoppedReplicating, StoppedReplicationInfoV2Message, decodeReplicas, encodeReplicas, isReplicationInfoV2Message, maxReplicas, } from "./replication.js";
+import { AbsoluteReplicas, AddedReplicationInfoV2Message, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, FullReplicationInfoV2Message, MinReplicas, ReplicationError, ReplicationInfoV2AppliedMessage, ReplicationPingMessage, RequestReplicationInfoMessage, RequestReplicationInfoV2AppliedMessage, RequestReplicationInfoV2Message, ResponseRoleMessage, StoppedReplicating, StoppedReplicationInfoV2Message, decodeReplicas, encodeReplicas, isReplicationInfoV2Message, maxReplicas, } from "./replication.js";
 import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
 import { createSyncronizer } from "./sync/factory.js";
 import { emitSyncProfileDuration, emitSyncProfileEvent, syncProfileStart, } from "./sync/profile.js";
@@ -150,11 +151,18 @@ const canUseOptionalNativeModuleImports = () => {
  * to avoid doubled slashes.
  */
 const joinNativeCoordinateDirectory = (nodeDirectory, fsSafeLogId) => `${nodeDirectory.replace(/[/\\]+$/, "")}/coordinates/${fsSafeLogId}`;
+const defaultNativeEntryBlockCompactMaxJournalBytes = 64 * 1024 * 1024;
 const createNativeDurableBlockStore = async (storage) => new AnyBlockStore(await storage.sublevel("blocks", {
-    // Strict mirrors remain WAL-backed across close. The available snapshot
-    // rewrite is not a crash-atomic generation protocol, so thresholds must
-    // not re-enable it behind this acknowledgement boundary.
+    // Strict mirrors remain WAL-backed. On POSIX Node, checkpoint only the
+    // historical suffix through the Rust store's fsync + atomic-rename path;
+    // browsers/custom backends and Windows retain the prior unbounded WAL until
+    // they expose an equally strong directory durability barrier.
     compactOnClose: false,
+    ...(storage.supportsCrashSafeJournalCheckpoint === true
+        ? {
+            compactMaxJournalBytes: defaultNativeEntryBlockCompactMaxJournalBytes,
+        }
+        : {}),
     // A native append is acknowledged only after this mirror resolves. The
     // Rust store's normal immutable fast path may resolve before its WAL write;
     // strict mode waits for the journal write and sync, closing the SIGKILL gap.
@@ -747,6 +755,13 @@ let SharedLog = (() => {
         _providerHandle;
         _isTrustedReplicator;
         _logProperties;
+        // A successful native signature batch is only an authorization input for the
+        // receive operation that requested it. Facts retain the exact verified bytes
+        // and are leased around the lower-log join; byte comparison prevents an
+        // earlier application callback from mutating a later entry underneath the
+        // batch result. The array form also makes overlapping leases for the same
+        // object safe without making verification sticky across retries.
+        _receiveSignatureVerificationFacts = new WeakMap();
         _closeController;
         _respondToIHaveTimeout;
         _checkedPrune;
@@ -1815,12 +1830,18 @@ let SharedLog = (() => {
                     this._peerSessions.isReceiveCleanupGateOpen(peerHash),
                 captureReplicationOwnershipLifecycle: () => this.captureReplicationOwnershipLifecycle(),
                 isReplicationOwnershipLifecycleActive: (controller) => this.isRepairLifecycleActive(controller),
+                supportsApplicationConfirmation: (peerHash, receiverTransportSession) => this._peerSyncCapabilitySessions.get(peerHash) ===
+                    receiverTransportSession &&
+                    ((this._peerSyncCapabilities.get(peerHash) ?? 0) &
+                        SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM) !==
+                        0,
             });
         }
         replicationInfoV2ReceiveCapabilities() {
             return (SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
                 SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
                 SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
+                SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM |
                 (this._logProperties?.sync?.rawExchangeHeads === true
                     ? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
                     : 0));
@@ -3807,7 +3828,11 @@ let SharedLog = (() => {
                 ? this.replicationController?.maxMemoryLimit
                 : undefined;
             try {
-                return await this.applyReplicationRole(rangeOrEntry, options);
+                const ranges = await this.applyReplicationRole(rangeOrEntry, options);
+                if (options?.confirm) {
+                    await this._v2Send.confirmLatest(typeof options.confirm === "object" ? options.confirm : undefined);
+                }
+                return ranges;
             }
             finally {
                 const storageObjective = this._isAdaptiveReplicating
@@ -3900,7 +3925,7 @@ let SharedLog = (() => {
             this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
             return this._replicate(range, options, replicationOwnershipLifecycleController);
         }
-        async unreplicate(rangeOrEntry) {
+        async unreplicate(rangeOrEntry, options) {
             const wasAdaptiveReplicating = this._isAdaptiveReplicating;
             const previousStorageObjective = wasAdaptiveReplicating
                 ? this.replicationController?.maxMemoryLimit
@@ -3908,7 +3933,11 @@ let SharedLog = (() => {
             this.throwIfCheckedPruneRemoveBlocksLocalOperation("replication range mutation");
             const replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle();
             try {
-                return await this._unreplicate(rangeOrEntry, replicationOwnershipLifecycleController);
+                const result = await this._unreplicate(rangeOrEntry, replicationOwnershipLifecycleController);
+                if (options?.confirm) {
+                    await this._v2Send.confirmLatest(typeof options.confirm === "object" ? options.confirm : undefined);
+                }
+                return result;
             }
             finally {
                 const storageObjective = this._isAdaptiveReplicating
@@ -9436,6 +9465,7 @@ let SharedLog = (() => {
                     : undefined,
             };
             this._logProperties = options;
+            this._receiveSignatureVerificationFacts = new WeakMap();
             this.domain = options?.domain
                 ? options.domain(this)
                 : createReplicationDomainHash("u64")(this);
@@ -11302,6 +11332,70 @@ let SharedLog = (() => {
                 }
             });
         }
+        receiveSignatureVerificationResult(entry) {
+            return getReceiveSignatureVerificationResult(this._receiveSignatureVerificationFacts, entry);
+        }
+        async preverifyReceiveSignaturesBatch(entries, profile) {
+            const candidates = entries.filter((entry) => !entry.createdLocally && !hasPreverifiedSignature(entry));
+            if (candidates.length < NATIVE_ED25519_VERIFY_BATCH_MIN_ENTRIES) {
+                return undefined;
+            }
+            let storageBytes;
+            let signableInputs;
+            try {
+                // Capture the callback-visible storage state and native signable inputs
+                // before the first await so the result is bound to one entry state.
+                storageBytes = candidates.map((entry) => entry.getStorageBytes().slice());
+                signableInputs =
+                    this.prepareReceiveNativeEd25519VerificationBatch(candidates);
+            }
+            catch {
+                return undefined;
+            }
+            if (!signableInputs) {
+                return undefined;
+            }
+            const verifyStartedAt = syncProfileStart(profile);
+            let verified;
+            try {
+                verified = await verifyEd25519Batch(signableInputs);
+            }
+            catch {
+                verified = undefined;
+            }
+            if (!verified || verified.length !== candidates.length) {
+                return undefined;
+            }
+            if (profile) {
+                emitSyncProfileDuration(profile, verifyStartedAt, {
+                    name: "sharedLog.canAppendBatch.verifySignatures",
+                    component: "shared-log",
+                    entries: candidates.length,
+                    messages: 1,
+                    details: {
+                        native: true,
+                        mode: "signable",
+                        programCanAppendDeferred: true,
+                    },
+                });
+            }
+            return candidates.map((entry, index) => ({
+                entry,
+                storageBytes: storageBytes[index],
+                verified: verified[index],
+            }));
+        }
+        prepareReceiveNativeEd25519VerificationBatch(entries) {
+            const inputs = this.prepareNativeEd25519VerificationBatch(entries);
+            return inputs?.map((input) => ({
+                signature: input.signature.slice(),
+                publicKey: input.publicKey.slice(),
+                message: input.message.slice(),
+            }));
+        }
+        async withReceiveSignatureVerificationFacts(facts, operation) {
+            return withSignatureVerificationFacts(this._receiveSignatureVerificationFacts, facts, operation);
+        }
         async canAppend(entry) {
             try {
                 if (!entry.meta.data) {
@@ -11313,9 +11407,16 @@ let SharedLog = (() => {
                     return false;
                 }
                 checkMinReplicasLimit(replicas);
-                // Locally-created entries were signed before append.
+                const receiveSignatureVerification = this.receiveSignatureVerificationResult(entry);
+                if (receiveSignatureVerification === false) {
+                    return false;
+                }
+                // Locally-created entries were signed before append. Raw prepared entries
+                // can carry a durable verifier fact from their decoder; plain receive facts
+                // are scoped to this lower-log join and checked against the exact bytes.
                 if (!entry.createdLocally &&
                     !hasPreverifiedSignature(entry) &&
+                    receiveSignatureVerification !== true &&
                     !(await entry.verifySignatures())) {
                     return false;
                 }
@@ -13453,6 +13554,9 @@ let SharedLog = (() => {
                             // entry views.
                             const programCanAppend = !!this._logProperties?.canAppend;
                             const programOnChange = !!this._logProperties?.onChange;
+                            const receiveSignatureVerificationFacts = programCanAppend
+                                ? await this.preverifyReceiveSignaturesBatch(materializeAllToMergeEntries(), syncProfile)
+                                : undefined;
                             const nativeBackboneCommitValidation = programCanAppend
                                 ? undefined
                                 : this.validatePreparedRawReceiveHeadsMetadataWithNativeBackbone(allToMerge, syncProfile, { decodedReplicaCounts: receiveReplicaCounts });
@@ -13617,29 +13721,34 @@ let SharedLog = (() => {
                                 const joinOnAppendHashes = programOnChange
                                     ? undefined
                                     : onAppendHashes;
-                                const joinedPreparedFacts = canUsePreparedAppendFacts &&
-                                    (await trustedLowerLog.joinPreparedAppendFactsBatch(preparedAppendFacts, {
-                                        __peerbitEntriesAlreadyMissing: true,
-                                        __peerbitCanAppendAlreadyValidated: true,
-                                        __peerbitDeferIndexWrite: true,
-                                        __peerbitOnAppendHashes: joinOnAppendHashes,
-                                        __peerbitProfile: syncProfile,
-                                        __peerbitNativePreparedJoinCommit: nativePreparedJoinCommit,
-                                        __peerbitNativePreparedJoinCommitValidatesPlan: nativePreparedJoinCommitValidatesPlan,
-                                        __peerbitOnPreparedJoinCommitted: nativePreparedJoinCommit
-                                            ? finishNativePreparedCoordinates
-                                            : undefined,
-                                    }));
-                                if (!joinedPreparedFacts) {
-                                    await trustedLowerLog.join(materializeAllToMergeEntries(), {
-                                        __peerbitBatchIndependent: true,
-                                        __peerbitEntriesAlreadyMissing: true,
-                                        __peerbitCanAppendAlreadyValidated: fallbackCanAppendAlreadyValidated,
-                                        __peerbitDeferIndexWrite: true,
-                                        __peerbitOnAppendHashes: joinOnAppendHashes,
-                                        __peerbitProfile: syncProfile,
-                                    });
-                                }
+                                let joinedPreparedFacts = false;
+                                const joinLowerLog = async () => {
+                                    joinedPreparedFacts =
+                                        canUsePreparedAppendFacts &&
+                                            (await trustedLowerLog.joinPreparedAppendFactsBatch(preparedAppendFacts, {
+                                                __peerbitEntriesAlreadyMissing: true,
+                                                __peerbitCanAppendAlreadyValidated: true,
+                                                __peerbitDeferIndexWrite: true,
+                                                __peerbitOnAppendHashes: joinOnAppendHashes,
+                                                __peerbitProfile: syncProfile,
+                                                __peerbitNativePreparedJoinCommit: nativePreparedJoinCommit,
+                                                __peerbitNativePreparedJoinCommitValidatesPlan: nativePreparedJoinCommitValidatesPlan,
+                                                __peerbitOnPreparedJoinCommitted: nativePreparedJoinCommit
+                                                    ? finishNativePreparedCoordinates
+                                                    : undefined,
+                                            }));
+                                    if (!joinedPreparedFacts) {
+                                        await trustedLowerLog.join(materializeAllToMergeEntries(), {
+                                            __peerbitBatchIndependent: true,
+                                            __peerbitEntriesAlreadyMissing: true,
+                                            __peerbitCanAppendAlreadyValidated: fallbackCanAppendAlreadyValidated,
+                                            __peerbitDeferIndexWrite: true,
+                                            __peerbitOnAppendHashes: joinOnAppendHashes,
+                                            __peerbitProfile: syncProfile,
+                                        });
+                                    }
+                                };
+                                await this.withReceiveSignatureVerificationFacts(receiveSignatureVerificationFacts, joinLowerLog);
                                 // A recursive lower-log join can resolve successfully while declining
                                 // an individual top-level entry (for example, when one of its parents
                                 // is temporarily unavailable). The public Log.join() API intentionally
@@ -13897,6 +14006,42 @@ let SharedLog = (() => {
                         this.clearRepairFrontierHashes(context.from.hashcode(), msg.hashes);
                         return;
                     }
+                    else if (msg instanceof RequestReplicationInfoV2AppliedMessage) {
+                        if (receiveSession?.phase === "open" &&
+                            this._peerSyncCapabilitySessions.get(receiveFromHash) ===
+                                context.message.header.session &&
+                            ((this._peerSyncCapabilities.get(receiveFromHash) ?? 0) &
+                                SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM) !==
+                                0) {
+                            const applied = this._v2Receive.confirmApplied(msg, {
+                                from: context.from,
+                                peerSession: receiveSession,
+                                receiveEpoch: receiveReplicationInfoReceiveEpoch,
+                                senderTransportSession: context.message.header.session,
+                            });
+                            if (applied) {
+                                await this.rpc.send(applied, {
+                                    mode: new SilentDelivery({
+                                        to: [context.from],
+                                        redundancy: 1,
+                                    }),
+                                    priority: CONVERGENCE_MESSAGE_PRIORITY,
+                                    signal: AbortSignal.any([
+                                        this._closeController.signal,
+                                        lane.ownershipLifecycleController.signal,
+                                    ]),
+                                });
+                            }
+                        }
+                        return;
+                    }
+                    else if (msg instanceof ReplicationInfoV2AppliedMessage) {
+                        this._v2Send.acceptApplied(msg, {
+                            from: context.from,
+                            receiverTransportSession: context.message.header.session,
+                        });
+                        return;
+                    }
                     else if (isReplicationInfoV2Message(msg)) {
                         await this.handleReplicationInfoV2Announcement(msg, laneRequestContext, lane);
                     }