@peerbit/shared-log 16.0.12 → 16.0.14

package/src/index.ts

@@ -164,6 +164,7 @@ import {
 	ResponseIPruneV2,
 	SYNC_CAPABILITY_RAW_EXCHANGE_HEADS,
 	SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY,
+	SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM,
 	SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE,
 	SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND,
 	StashBackedRawExchangeHeadsMessage,
@@ -231,6 +232,11 @@ import {
 	shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary,
 	toRebalance,
 } from "./ranges.js";
+import {
+	type ReceiveSignatureVerificationFact,
+	receiveSignatureVerificationResult as getReceiveSignatureVerificationResult,
+	withReceiveSignatureVerificationFacts as withSignatureVerificationFacts,
+} from "./receive-signature-verification.js";
 import { ReplicationAnnouncementCoordinator } from "./replication-announcement.js";
 import {
 	type ReplicationDomainHash,
@@ -251,7 +257,10 @@ import type {
 	FullReplicationInfoMutation,
 } from "./replication-info-mutation.js";
 import { ReplicationInfoV2ReceiveCoordinator } from "./replication-info-v2-receive.js";
-import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
+import {
+	type ReplicationInfoV2ConfirmationOptions,
+	ReplicationInfoV2SendCoordinator,
+} from "./replication-info-v2-send.js";
 import {
 	type ReplicationStatus,
 	classifyReplicationStatus,
@@ -264,10 +273,12 @@ import {
 	FullReplicationInfoV2Message,
 	MinReplicas,
 	ReplicationError,
+	ReplicationInfoV2AppliedMessage,
 	type ReplicationInfoV2Message,
 	type ReplicationLimits,
 	ReplicationPingMessage,
 	RequestReplicationInfoMessage,
+	RequestReplicationInfoV2AppliedMessage,
 	RequestReplicationInfoV2Message,
 	ResponseRoleMessage,
 	StoppedReplicating,
@@ -487,25 +498,36 @@ const joinNativeCoordinateDirectory = (
 	`${nodeDirectory.replace(/[/\\]+$/, "")}/coordinates/${fsSafeLogId}`;
 
 type DurableBlockSublevelStore = {
+	readonly supportsCrashSafeJournalCheckpoint?: boolean;
 	sublevel(
 		name: string,
 		options?: {
 			compactOnClose?: boolean;
 			compactOnCloseMinJournalBytes?: number;
+			compactMaxJournalBytes?: number;
 			durability?: "normal" | "strict";
 		},
 	): MaybePromise<AnyStore>;
 };
 
+const defaultNativeEntryBlockCompactMaxJournalBytes = 64 * 1024 * 1024;
+
 const createNativeDurableBlockStore = async (
 	storage: DurableBlockSublevelStore,
 ): Promise<AnyBlockStore> =>
 	new AnyBlockStore(
 		await storage.sublevel("blocks", {
-			// Strict mirrors remain WAL-backed across close. The available snapshot
-			// rewrite is not a crash-atomic generation protocol, so thresholds must
-			// not re-enable it behind this acknowledgement boundary.
+			// Strict mirrors remain WAL-backed. On POSIX Node, checkpoint only the
+			// historical suffix through the Rust store's fsync + atomic-rename path;
+			// browsers/custom backends and Windows retain the prior unbounded WAL until
+			// they expose an equally strong directory durability barrier.
 			compactOnClose: false,
+			...(storage.supportsCrashSafeJournalCheckpoint === true
+				? {
+						compactMaxJournalBytes:
+							defaultNativeEntryBlockCompactMaxJournalBytes,
+					}
+				: {}),
 			// A native append is acknowledged only after this mirror resolves. The
 			// Rust store's normal immutable fast path may resolve before its WAL write;
 			// strict mode waits for the journal write and sync, closing the SIGKILL gap.
@@ -1065,6 +1087,10 @@ export type ReplicationOptions<R extends "u32" | "u64" = any> =
 	| NewReplicationOptions<R>
 	| ExistingReplicationOptions<R>;
 
+/** Opt-in proof that a local replication-role update applied remotely. */
+export type ReplicationConfirmationOptions =
+	ReplicationInfoV2ConfirmationOptions;
+
 export { BlocksMessage };
 
 const isAdaptiveReplicatorOption = (
@@ -2179,6 +2205,16 @@ export class SharedLog<
 	private _logProperties?: LogProperties<T> &
 		LogEvents<T> &
 		SharedLogOptions<T, D, R>;
+	// A successful native signature batch is only an authorization input for the
+	// receive operation that requested it. Facts retain the exact verified bytes
+	// and are leased around the lower-log join; byte comparison prevents an
+	// earlier application callback from mutating a later entry underneath the
+	// batch result. The array form also makes overlapping leases for the same
+	// object safe without making verification sticky across retries.
+	private _receiveSignatureVerificationFacts = new WeakMap<
+		Entry<T>,
+		ReceiveSignatureVerificationFact<T>[]
+	>();
 	private _closeController!: AbortController;
 	private _respondToIHaveTimeout!: any;
 	private _checkedPrune!: CheckedPruneCoordinator<T, R>;
@@ -3561,6 +3597,12 @@ export class SharedLog<
 				this.captureReplicationOwnershipLifecycle(),
 			isReplicationOwnershipLifecycleActive: (controller) =>
 				this.isRepairLifecycleActive(controller),
+			supportsApplicationConfirmation: (peerHash, receiverTransportSession) =>
+				this._peerSyncCapabilitySessions.get(peerHash) ===
+					receiverTransportSession &&
+				((this._peerSyncCapabilities.get(peerHash) ?? 0) &
+					SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM) !==
+					0,
 		});
 	}
 
@@ -3569,6 +3611,7 @@ export class SharedLog<
 			SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
 			SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
 			SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
+			SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM |
 			(this._logProperties?.sync?.rawExchangeHeads === true
 				? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
 				: 0)
@@ -6187,6 +6230,8 @@ export class SharedLog<
 			checkDuplicates?: boolean;
 			rebalance?: boolean;
 			mergeSegments?: boolean;
+			/** Wait for explicit remote durable application; omitted is unchanged. */
+			confirm?: boolean | ReplicationConfirmationOptions;
 			announce?: (
 				msg: FullReplicationInfoMutation | AddedReplicationInfoMutation,
 			) => void;
@@ -6197,7 +6242,13 @@ export class SharedLog<
 			? this.replicationController?.maxMemoryLimit
 			: undefined;
 		try {
-			return await this.applyReplicationRole(rangeOrEntry, options);
+			const ranges = await this.applyReplicationRole(rangeOrEntry, options);
+			if (options?.confirm) {
+				await this._v2Send.confirmLatest(
+					typeof options.confirm === "object" ? options.confirm : undefined,
+				);
+			}
+			return ranges;
 		} finally {
 			const storageObjective = this._isAdaptiveReplicating
 				? this.replicationController?.maxMemoryLimit
@@ -6329,7 +6380,10 @@ export class SharedLog<
 		);
 	}
 
-	async unreplicate(rangeOrEntry?: Entry<T> | { id: Uint8Array }[]) {
+	async unreplicate(
+		rangeOrEntry?: Entry<T> | { id: Uint8Array }[],
+		options?: { confirm?: boolean | ReplicationConfirmationOptions },
+	) {
 		const wasAdaptiveReplicating = this._isAdaptiveReplicating;
 		const previousStorageObjective = wasAdaptiveReplicating
 			? this.replicationController?.maxMemoryLimit
@@ -6340,10 +6394,16 @@ export class SharedLog<
 		const replicationOwnershipLifecycleController =
 			this.captureReplicationOwnershipLifecycle();
 		try {
-			return await this._unreplicate(
+			const result = await this._unreplicate(
 				rangeOrEntry,
 				replicationOwnershipLifecycleController,
 			);
+			if (options?.confirm) {
+				await this._v2Send.confirmLatest(
+					typeof options.confirm === "object" ? options.confirm : undefined,
+				);
+			}
+			return result;
 		} finally {
 			const storageObjective = this._isAdaptiveReplicating
 				? this.replicationController?.maxMemoryLimit
@@ -14600,6 +14660,7 @@ export class SharedLog<
 				: undefined,
 		};
 		this._logProperties = options;
+		this._receiveSignatureVerificationFacts = new WeakMap();
 
 		this.domain = options?.domain
 			? (options.domain(this) as unknown as D)
@@ -17039,6 +17100,93 @@ export class SharedLog<
 		});
 	}
 
+	private receiveSignatureVerificationResult(
+		entry: Entry<T>,
+	): boolean | undefined {
+		return getReceiveSignatureVerificationResult(
+			this._receiveSignatureVerificationFacts,
+			entry,
+		);
+	}
+
+	private async preverifyReceiveSignaturesBatch(
+		entries: Entry<T>[],
+		profile?: SyncProfileFn,
+	): Promise<ReceiveSignatureVerificationFact<T>[] | undefined> {
+		const candidates = entries.filter(
+			(entry) => !entry.createdLocally && !hasPreverifiedSignature(entry),
+		);
+		if (candidates.length < NATIVE_ED25519_VERIFY_BATCH_MIN_ENTRIES) {
+			return undefined;
+		}
+
+		let storageBytes: Uint8Array[];
+		let signableInputs: Ed25519VerifyBatchInput[] | undefined;
+		try {
+			// Capture the callback-visible storage state and native signable inputs
+			// before the first await so the result is bound to one entry state.
+			storageBytes = candidates.map((entry) => entry.getStorageBytes().slice());
+			signableInputs =
+				this.prepareReceiveNativeEd25519VerificationBatch(candidates);
+		} catch {
+			return undefined;
+		}
+		if (!signableInputs) {
+			return undefined;
+		}
+
+		const verifyStartedAt = syncProfileStart(profile);
+		let verified: boolean[] | undefined;
+		try {
+			verified = await verifyEd25519Batch(signableInputs);
+		} catch {
+			verified = undefined;
+		}
+		if (!verified || verified.length !== candidates.length) {
+			return undefined;
+		}
+		if (profile) {
+			emitSyncProfileDuration(profile, verifyStartedAt, {
+				name: "sharedLog.canAppendBatch.verifySignatures",
+				component: "shared-log",
+				entries: candidates.length,
+				messages: 1,
+				details: {
+					native: true,
+					mode: "signable",
+					programCanAppendDeferred: true,
+				},
+			});
+		}
+		return candidates.map((entry, index) => ({
+			entry,
+			storageBytes: storageBytes[index]!,
+			verified: verified[index]!,
+		}));
+	}
+
+	private prepareReceiveNativeEd25519VerificationBatch(
+		entries: Entry<T>[],
+	): Ed25519VerifyBatchInput[] | undefined {
+		const inputs = this.prepareNativeEd25519VerificationBatch(entries);
+		return inputs?.map((input) => ({
+			signature: input.signature.slice(),
+			publicKey: input.publicKey.slice(),
+			message: input.message.slice(),
+		}));
+	}
+
+	private async withReceiveSignatureVerificationFacts<R>(
+		facts: ReceiveSignatureVerificationFact<T>[] | undefined,
+		operation: () => Promise<R>,
+	): Promise<R> {
+		return withSignatureVerificationFacts(
+			this._receiveSignatureVerificationFacts,
+			facts,
+			operation,
+		);
+	}
+
 	async canAppend(entry: Entry<T>) {
 		try {
 			if (!entry.meta.data) {
@@ -17052,10 +17200,19 @@ export class SharedLog<
 
 			checkMinReplicasLimit(replicas);
 
-			// Locally-created entries were signed before append.
+			const receiveSignatureVerification =
+				this.receiveSignatureVerificationResult(entry);
+			if (receiveSignatureVerification === false) {
+				return false;
+			}
+
+			// Locally-created entries were signed before append. Raw prepared entries
+			// can carry a durable verifier fact from their decoder; plain receive facts
+			// are scoped to this lower-log join and checked against the exact bytes.
 			if (
 				!entry.createdLocally &&
 				!hasPreverifiedSignature(entry) &&
+				receiveSignatureVerification !== true &&
 				!(await entry.verifySignatures())
 			) {
 				return false;
@@ -19608,6 +19765,12 @@ export class SharedLog<
 						// entry views.
 						const programCanAppend = !!this._logProperties?.canAppend;
 						const programOnChange = !!this._logProperties?.onChange;
+						const receiveSignatureVerificationFacts = programCanAppend
+							? await this.preverifyReceiveSignaturesBatch(
+									materializeAllToMergeEntries(),
+									syncProfile,
+								)
+							: undefined;
 						const nativeBackboneCommitValidation = programCanAppend
 							? undefined
 							: this.validatePreparedRawReceiveHeadsMetadataWithNativeBackbone(
@@ -19811,35 +19974,43 @@ export class SharedLog<
 							const joinOnAppendHashes = programOnChange
 								? undefined
 								: onAppendHashes;
-							const joinedPreparedFacts =
-								canUsePreparedAppendFacts &&
-								(await trustedLowerLog.joinPreparedAppendFactsBatch(
-									preparedAppendFacts,
-									{
+							let joinedPreparedFacts = false;
+							const joinLowerLog = async () => {
+								joinedPreparedFacts =
+									canUsePreparedAppendFacts &&
+									(await trustedLowerLog.joinPreparedAppendFactsBatch(
+										preparedAppendFacts,
+										{
+											__peerbitEntriesAlreadyMissing: true,
+											__peerbitCanAppendAlreadyValidated: true,
+											__peerbitDeferIndexWrite: true,
+											__peerbitOnAppendHashes: joinOnAppendHashes,
+											__peerbitProfile: syncProfile,
+											__peerbitNativePreparedJoinCommit:
+												nativePreparedJoinCommit,
+											__peerbitNativePreparedJoinCommitValidatesPlan:
+												nativePreparedJoinCommitValidatesPlan,
+											__peerbitOnPreparedJoinCommitted: nativePreparedJoinCommit
+												? finishNativePreparedCoordinates
+												: undefined,
+										},
+									));
+								if (!joinedPreparedFacts) {
+									await trustedLowerLog.join(materializeAllToMergeEntries(), {
+										__peerbitBatchIndependent: true,
 										__peerbitEntriesAlreadyMissing: true,
-										__peerbitCanAppendAlreadyValidated: true,
+										__peerbitCanAppendAlreadyValidated:
+											fallbackCanAppendAlreadyValidated,
 										__peerbitDeferIndexWrite: true,
 										__peerbitOnAppendHashes: joinOnAppendHashes,
 										__peerbitProfile: syncProfile,
-										__peerbitNativePreparedJoinCommit: nativePreparedJoinCommit,
-										__peerbitNativePreparedJoinCommitValidatesPlan:
-											nativePreparedJoinCommitValidatesPlan,
-										__peerbitOnPreparedJoinCommitted: nativePreparedJoinCommit
-											? finishNativePreparedCoordinates
-											: undefined,
-									},
-								));
-							if (!joinedPreparedFacts) {
-								await trustedLowerLog.join(materializeAllToMergeEntries(), {
-									__peerbitBatchIndependent: true,
-									__peerbitEntriesAlreadyMissing: true,
-									__peerbitCanAppendAlreadyValidated:
-										fallbackCanAppendAlreadyValidated,
-									__peerbitDeferIndexWrite: true,
-									__peerbitOnAppendHashes: joinOnAppendHashes,
-									__peerbitProfile: syncProfile,
-								});
-							}
+									});
+								}
+							};
+							await this.withReceiveSignatureVerificationFacts(
+								receiveSignatureVerificationFacts,
+								joinLowerLog,
+							);
 							// A recursive lower-log join can resolve successfully while declining
 							// an individual top-level entry (for example, when one of its parents
 							// is temporarily unavailable). The public Log.join() API intentionally
@@ -20153,6 +20324,42 @@ export class SharedLog<
 					this.markEntriesKnownByPeer(msg.hashes, context.from.hashcode());
 					this.clearRepairFrontierHashes(context.from.hashcode(), msg.hashes);
 					return;
+				} else if (msg instanceof RequestReplicationInfoV2AppliedMessage) {
+					if (
+						receiveSession?.phase === "open" &&
+						this._peerSyncCapabilitySessions.get(receiveFromHash) ===
+							context.message.header.session &&
+						((this._peerSyncCapabilities.get(receiveFromHash) ?? 0) &
+							SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM) !==
+							0
+					) {
+						const applied = this._v2Receive.confirmApplied(msg, {
+							from: context.from,
+							peerSession: receiveSession,
+							receiveEpoch: receiveReplicationInfoReceiveEpoch,
+							senderTransportSession: context.message.header.session,
+						});
+						if (applied) {
+							await this.rpc.send(applied, {
+								mode: new SilentDelivery({
+									to: [context.from],
+									redundancy: 1,
+								}),
+								priority: CONVERGENCE_MESSAGE_PRIORITY,
+								signal: AbortSignal.any([
+									this._closeController.signal,
+									lane.ownershipLifecycleController.signal,
+								]),
+							});
+						}
+					}
+					return;
+				} else if (msg instanceof ReplicationInfoV2AppliedMessage) {
+					this._v2Send.acceptApplied(msg, {
+						from: context.from,
+						receiverTransportSession: context.message.header.session,
+					});
+					return;
 				} else if (isReplicationInfoV2Message(msg)) {
 					await this.handleReplicationInfoV2Announcement(
 						msg,

package/src/receive-signature-verification.ts

@@ -0,0 +1,66 @@
+import type { Entry } from "@peerbit/log";
+import { equals as bytesEqual } from "uint8arrays";
+
+export type ReceiveSignatureVerificationFact<T> = {
+	entry: Entry<T>;
+	storageBytes: Uint8Array;
+	verified: boolean;
+};
+
+export const receiveSignatureVerificationResult = <T>(
+	factsByEntry: WeakMap<Entry<T>, ReceiveSignatureVerificationFact<T>[]>,
+	entry: Entry<T>,
+): boolean | undefined => {
+	const facts = factsByEntry.get(entry);
+	if (!facts || facts.length === 0) {
+		return undefined;
+	}
+	let storageBytes: Uint8Array;
+	try {
+		storageBytes = entry.getStorageBytes();
+	} catch {
+		return undefined;
+	}
+	for (let i = facts.length - 1; i >= 0; i--) {
+		const fact = facts[i]!;
+		if (bytesEqual(storageBytes, fact.storageBytes)) {
+			return fact.verified;
+		}
+	}
+	return undefined;
+};
+
+export const withReceiveSignatureVerificationFacts = async <T, R>(
+	factsByEntry: WeakMap<Entry<T>, ReceiveSignatureVerificationFact<T>[]>,
+	facts: ReceiveSignatureVerificationFact<T>[] | undefined,
+	operation: () => Promise<R>,
+): Promise<R> => {
+	if (!facts || facts.length === 0) {
+		return operation();
+	}
+	for (const fact of facts) {
+		const active = factsByEntry.get(fact.entry);
+		if (active) {
+			active.push(fact);
+		} else {
+			factsByEntry.set(fact.entry, [fact]);
+		}
+	}
+	try {
+		return await operation();
+	} finally {
+		for (const fact of facts) {
+			const active = factsByEntry.get(fact.entry);
+			if (!active) {
+				continue;
+			}
+			const index = active.indexOf(fact);
+			if (index >= 0) {
+				active.splice(index, 1);
+			}
+			if (active.length === 0) {
+				factsByEntry.delete(fact.entry);
+			}
+		}
+	}
+};

package/src/replication-info-v2-receive.ts

@@ -10,7 +10,9 @@ import {
 	AddedReplicationSegmentMessage,
 	AllReplicatingSegmentsMessage,
 	FullReplicationInfoV2Message,
+	ReplicationInfoV2AppliedMessage,
 	type ReplicationInfoV2Message,
+	RequestReplicationInfoV2AppliedMessage,
 	RequestReplicationInfoV2Message,
 	StoppedReplicating,
 	StoppedReplicationInfoV2Message,
@@ -1250,6 +1252,46 @@ export class ReplicationInfoV2ReceiveCoordinator {
 		return true;
 	}
 
+	/**
+	 * Answer only after the exact authenticated receive generation has committed
+	 * the queried sequence (or a newer one). No pending query is retained: the
+	 * sender's bounded retry loop reissues an authoritative snapshot and query.
+	 */
+	confirmApplied(
+		request: RequestReplicationInfoV2AppliedMessage,
+		properties: {
+			from: PublicSignKey;
+			peerSession: object;
+			receiveEpoch: object | null;
+			senderTransportSession: bigint;
+		},
+	): ReplicationInfoV2AppliedMessage | undefined {
+		const state = this._receiveStates.get(properties.from.hashcode());
+		if (
+			!state ||
+			state.peerSession !== properties.peerSession ||
+			state.receiveEpoch !== properties.receiveEpoch ||
+			state.senderTransportSession !== properties.senderTransportSession ||
+			!state.target.equals(properties.from) ||
+			!state.receiverBinding ||
+			!bytesEqual(request.receiverChallenge, state.receiverBinding) ||
+			!state.senderEpoch ||
+			!bytesEqual(request.senderEpoch, state.senderEpoch) ||
+			request.sequence <= 0n ||
+			state.lastSequence === undefined ||
+			state.lastSequence < request.sequence ||
+			!this.isStateCurrent(state)
+		) {
+			return undefined;
+		}
+		return new ReplicationInfoV2AppliedMessage({
+			receiverChallenge: request.receiverChallenge.slice(),
+			senderEpoch: request.senderEpoch.slice(),
+			sequence: request.sequence,
+			revision: request.revision,
+		});
+	}
+
 	requireFullAfterFailure(
 		admission: ReplicationInfoV2ReceiveAdmission,
 	): boolean {