@peerbit/shared-log 13.2.12 → 13.2.14

package/src/checked-prune.ts

@@ -10,10 +10,18 @@ export type CheckedPruneEntry<T, R extends "u32" | "u64"> =
 	| EntryReplicated<R>;
 
 export type CheckedPrunePendingDelete = {
+	requestId: Uint8Array;
+	retryOnInvalidation?: boolean;
 	promise: DeferredPromise<void>;
 	clear: () => void;
-	resolve: (publicKeyHash: string) => Promise<void> | void;
-	reject(reason: any): Promise<void> | void;
+	resolve: (
+		publicKeyHash: string,
+		requestId: Uint8Array,
+	) => Promise<void> | void;
+	reject(
+		reason: any,
+		options?: { preserveRetry?: boolean },
+	): Promise<void> | void;
 };
 
 export type CheckedPruneRetryState<T, R extends "u32" | "u64"> = {
@@ -23,6 +31,18 @@ export type CheckedPruneRetryState<T, R extends "u32" | "u64"> = {
 	leaders: CheckedPruneLeaderMap | Set<string>;
 };
 
+export type CheckedPruneInvalidatedGeneration<T, R extends "u32" | "u64"> = {
+	hash: string;
+	pending: CheckedPrunePendingDelete;
+	entry: CheckedPruneEntry<T, R>;
+	leaders: CheckedPruneLeaderMap | Set<string>;
+};
+
+export type CheckedPruneRestartCandidate<T, R extends "u32" | "u64"> = Omit<
+	CheckedPruneInvalidatedGeneration<T, R>,
+	"pending"
+>;
+
 type CheckedPruneSessionPhase =
 	| "candidate"
 	| "requested"
@@ -40,6 +60,7 @@ type CheckedPruneSession<T, R extends "u32" | "u64"> = {
 	retry?: CheckedPruneRetryState<T, R>;
 	contacted: Set<string>;
 	confirmed: Set<string>;
+	revoked: Set<string>;
 };
 
 export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
@@ -48,6 +69,10 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 	readonly responseReplicatorSet = new Map<string, Set<string>>();
 	readonly retries = new Map<string, CheckedPruneRetryState<T, R>>();
 	private readonly sessions = new Map<string, CheckedPruneSession<T, R>>();
+	private readonly grantSends = new Map<string, Set<Promise<void>>>();
+	private readonly peerRemovalFences = new Map<string, number>();
+	private readonly restartReservations = new Map<string, object>();
+	private readonly candidateTokens = new Map<string, object>();
 
 	private getOrCreateSession(hash: string): CheckedPruneSession<T, R> {
 		let session = this.sessions.get(hash);
@@ -56,6 +81,7 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 				phase: "candidate",
 				contacted: new Set(),
 				confirmed: new Set(),
+				revoked: new Set(),
 			};
 			this.sessions.set(hash, session);
 		}
@@ -82,7 +108,8 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 			this.pendingDeletes.has(hash) ||
 			this.requestIPruneSent.has(hash) ||
 			this.responseReplicatorSet.has(hash) ||
-			this.retries.has(hash)
+			this.retries.has(hash) ||
+			this.candidateTokens.has(hash)
 		) {
 			return;
 		}
@@ -94,10 +121,27 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 		entry: CheckedPruneEntry<T, R>,
 		leaders: CheckedPruneLeaderMap | Set<string>,
 	) {
+		this.restartReservations.delete(hash);
 		const session = this.remember(hash, entry, leaders);
 		if (!this.hasActiveWork(hash)) {
 			session.phase = "candidate";
 		}
+		const token = {};
+		this.candidateTokens.set(hash, token);
+		return token;
+	}
+
+	isCandidateTokenCurrent(hash: string, token: object) {
+		return this.candidateTokens.get(hash) === token;
+	}
+
+	hasCandidate(hash: string) {
+		return this.candidateTokens.has(hash);
+	}
+
+	invalidateCandidateToken(hash: string) {
+		this.candidateTokens.delete(hash);
+		this.deleteSessionIfIdle(hash);
 	}
 
 	hasActiveWork(hash: string) {
@@ -105,7 +149,8 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 			this.pendingDeletes.has(hash) ||
 			this.requestIPruneSent.has(hash) ||
 			this.responseReplicatorSet.has(hash) ||
-			this.retries.has(hash)
+			this.retries.has(hash) ||
+			this.candidateTokens.has(hash)
 		);
 	}
 
@@ -117,15 +162,39 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 		return this.pendingDeletes.has(hash);
 	}
 
+	getRestartCandidate(
+		hash: string,
+	): CheckedPruneRestartCandidate<T, R> | undefined {
+		const session = this.sessions.get(hash);
+		if (!session?.entry || !session.leaders) {
+			return undefined;
+		}
+		return {
+			hash,
+			entry: session.entry,
+			leaders:
+				session.leaders instanceof Map
+					? new Map(session.leaders)
+					: new Set(session.leaders),
+		};
+	}
+
 	setPendingDelete(
 		hash: string,
 		pending: CheckedPrunePendingDelete,
 		entry: CheckedPruneEntry<T, R>,
 		leaders: CheckedPruneLeaderMap | Set<string>,
 	) {
+		this.restartReservations.delete(hash);
+		this.candidateTokens.delete(hash);
+		this.requestIPruneSent.delete(hash);
+		this.responseReplicatorSet.delete(hash);
 		this.pendingDeletes.set(hash, pending);
 		const session = this.remember(hash, entry, leaders);
 		session.pending = pending;
+		session.contacted.clear();
+		session.confirmed.clear();
+		session.revoked.clear();
 		session.phase = "requested";
 	}
 
@@ -133,9 +202,14 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 		const current = this.pendingDeletes.get(hash);
 		if (!pending || current === pending) {
 			this.pendingDeletes.delete(hash);
+			this.requestIPruneSent.delete(hash);
+			this.responseReplicatorSet.delete(hash);
 			const session = this.sessions.get(hash);
 			if (session && session.pending === current) {
 				session.pending = undefined;
+				session.contacted.clear();
+				session.confirmed.clear();
+				session.revoked.clear();
 			}
 			this.deleteSessionIfIdle(hash);
 		}
@@ -150,6 +224,8 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 	}
 
 	setRetry(hash: string, state: CheckedPruneRetryState<T, R>) {
+		this.restartReservations.delete(hash);
+		this.candidateTokens.delete(hash);
 		this.retries.set(hash, state);
 		const session = this.remember(hash, state.entry, state.leaders);
 		session.retry = state;
@@ -157,6 +233,8 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 	}
 
 	clearRetry(hash: string) {
+		this.restartReservations.delete(hash);
+		this.candidateTokens.delete(hash);
 		const state = this.retries.get(hash);
 		if (state?.timer) {
 			clearTimeout(state.timer);
@@ -178,34 +256,78 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 		return state;
 	}
 
-	addRequestSent(hash: string, peer: string) {
+	isCurrentRequest(hash: string, pending: CheckedPrunePendingDelete) {
+		return this.pendingDeletes.get(hash) === pending;
+	}
+
+	isCurrentRequestId(
+		hash: string,
+		pending: CheckedPrunePendingDelete,
+		requestId: Uint8Array,
+	) {
+		if (!this.isCurrentRequest(hash, pending)) {
+			return false;
+		}
+		if (pending.requestId.byteLength !== requestId.byteLength) {
+			return false;
+		}
+		return pending.requestId.every(
+			(value, index) => value === requestId[index],
+		);
+	}
+
+	addRequestSent(
+		hash: string,
+		peer: string,
+		pending: CheckedPrunePendingDelete,
+	) {
+		if (!this.isCurrentRequest(hash, pending)) {
+			return false;
+		}
+		const session = this.getOrCreateSession(hash);
+		if (session.revoked.has(peer)) {
+			return false;
+		}
 		let set = this.requestIPruneSent.get(hash);
 		if (!set) {
 			set = new Set();
 			this.requestIPruneSent.set(hash, set);
 		}
 		set.add(peer);
-		const session = this.getOrCreateSession(hash);
 		session.contacted.add(peer);
 		if (session.phase === "candidate") {
 			session.phase = "requested";
 		}
+		return true;
 	}
 
 	removeRequestSent(hash: string, peer?: string) {
 		if (!peer) {
-			this.requestIPruneSent.delete(hash);
 			const session = this.sessions.get(hash);
+			if (session?.pending) {
+				for (const contactedPeer of session.contacted) {
+					session.revoked.add(contactedPeer);
+				}
+			}
+			this.requestIPruneSent.delete(hash);
 			session?.contacted.clear();
+			this.clearConfirmedReplicators(hash);
 			this.deleteSessionIfIdle(hash);
 			return;
 		}
+		const session = this.sessions.get(hash);
+		if (session?.pending) {
+			// A contact removed during this pending generation must never become
+			// authorized again by a delayed receipt carrying the old generation ID.
+			session.revoked.add(peer);
+		}
 		const set = this.requestIPruneSent.get(hash);
 		if (!set) {
+			this.removeConfirmedReplicator(hash, peer);
 			return;
 		}
 		set.delete(peer);
-		const session = this.sessions.get(hash);
+		this.removeConfirmedReplicator(hash, peer);
 		session?.contacted.delete(peer);
 		if (set.size === 0) {
 			this.requestIPruneSent.delete(hash);
@@ -214,15 +336,23 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 	}
 
 	removeRequestsSent(hashes: Iterable<string>, peer?: string) {
-		if (this.requestIPruneSent.size === 0) {
-			return;
-		}
 		for (const hash of hashes) {
 			this.removeRequestSent(hash, peer);
 		}
 	}
 
-	addConfirmedReplicator(hash: string, peer: string) {
+	addConfirmedReplicator(
+		hash: string,
+		peer: string,
+		pending: CheckedPrunePendingDelete,
+		requestId: Uint8Array,
+	) {
+		if (
+			!this.isCurrentRequestId(hash, pending, requestId) ||
+			!this.requestIPruneSent.get(hash)?.has(peer)
+		) {
+			return undefined;
+		}
 		let set = this.responseReplicatorSet.get(hash);
 		if (!set) {
 			set = new Set();
@@ -282,24 +412,178 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 		return this.requestIPruneSent.get(hash);
 	}
 
-	markRemoving(hash: string) {
+	hasRevokedLeader(
+		hash: string,
+		pending: CheckedPrunePendingDelete,
+		leaders: CheckedPruneLeaderMap | Set<string>,
+	) {
+		if (!this.isCurrentRequest(hash, pending)) {
+			return false;
+		}
+		const revoked = this.sessions.get(hash)?.revoked;
+		if (!revoked || revoked.size === 0) {
+			return false;
+		}
+		for (const leader of leaders.keys()) {
+			if (revoked.has(leader)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	trackGrantSend(hashes: Iterable<string>, send: Promise<void>) {
+		const uniqueHashes = [...new Set(hashes)];
+		const observed = send.then(
+			() => undefined,
+			() => undefined,
+		);
+		for (const hash of uniqueHashes) {
+			let sends = this.grantSends.get(hash);
+			if (!sends) {
+				sends = new Set();
+				this.grantSends.set(hash, sends);
+			}
+			sends.add(observed);
+		}
+		void observed.finally(() => {
+			for (const hash of uniqueHashes) {
+				const sends = this.grantSends.get(hash);
+				sends?.delete(observed);
+				if (sends?.size === 0) {
+					this.grantSends.delete(hash);
+				}
+			}
+		});
+		return observed;
+	}
+
+	waitForGrantSends(hash: string): Promise<void> | undefined {
+		const sends = this.grantSends.get(hash);
+		if (!sends || sends.size === 0) {
+			return undefined;
+		}
+		return Promise.all(sends).then(() => undefined);
+	}
+
+	reserveRestart(hash: string) {
+		const reservation = {};
+		this.restartReservations.set(hash, reservation);
+		return reservation;
+	}
+
+	consumeRestartReservation(hash: string, reservation: object) {
+		if (this.restartReservations.get(hash) !== reservation) {
+			return false;
+		}
+		this.restartReservations.delete(hash);
+		return true;
+	}
+
+	cancelRestartReservation(hash: string, reservation: object) {
+		if (this.restartReservations.get(hash) === reservation) {
+			this.restartReservations.delete(hash);
+		}
+	}
+
+	hasRestartReservation(hash: string) {
+		return this.restartReservations.has(hash);
+	}
+
+	getExactConfirmedReplicators(
+		hash: string,
+		pending: CheckedPrunePendingDelete,
+	) {
+		const exact = new Set<string>();
+		if (!this.isCurrentRequest(hash, pending)) {
+			return exact;
+		}
+		const contacted = this.requestIPruneSent.get(hash);
+		const confirmed = this.responseReplicatorSet.get(hash);
+		const revoked = this.sessions.get(hash)?.revoked;
+		if (!contacted || !confirmed) {
+			return exact;
+		}
+		for (const peer of confirmed) {
+			if (
+				contacted.has(peer) &&
+				!revoked?.has(peer) &&
+				!this.isPeerRemovalFenced(peer)
+			) {
+				exact.add(peer);
+			}
+		}
+		return exact;
+	}
+
+	fencePeerRemoval(peer: string) {
+		this.peerRemovalFences.set(
+			peer,
+			(this.peerRemovalFences.get(peer) ?? 0) + 1,
+		);
+		let released = false;
+		return () => {
+			if (released) {
+				return;
+			}
+			released = true;
+			const remaining = (this.peerRemovalFences.get(peer) ?? 1) - 1;
+			if (remaining > 0) {
+				this.peerRemovalFences.set(peer, remaining);
+			} else {
+				this.peerRemovalFences.delete(peer);
+			}
+		};
+	}
+
+	isPeerRemovalFenced(peer: string) {
+		return (this.peerRemovalFences.get(peer) ?? 0) > 0;
+	}
+
+	markRemoving(hash: string, pending?: CheckedPrunePendingDelete) {
+		if (pending && !this.isCurrentRequest(hash, pending)) {
+			return false;
+		}
 		const session = this.getOrCreateSession(hash);
 		session.phase = "removing";
+		return true;
 	}
 
-	markDone(hash: string) {
+	markDone(hash: string, pending?: CheckedPrunePendingDelete) {
+		if (pending && !this.isCurrentRequest(hash, pending)) {
+			return false;
+		}
 		const session = this.sessions.get(hash);
 		if (session) {
 			session.phase = "done";
 		}
+		this.restartReservations.delete(hash);
+		this.candidateTokens.delete(hash);
 		this.pendingDeletes.delete(hash);
 		this.requestIPruneSent.delete(hash);
 		this.responseReplicatorSet.delete(hash);
 		this.clearRetry(hash);
 		this.sessions.delete(hash);
+		return true;
 	}
 
-	markCancelled(hash: string, options?: { preserveRetry?: boolean }) {
+	markCancelled(
+		hash: string,
+		pendingOrOptions?: CheckedPrunePendingDelete | { preserveRetry?: boolean },
+		options?: { preserveRetry?: boolean },
+	) {
+		const pending =
+			pendingOrOptions && "promise" in pendingOrOptions
+				? pendingOrOptions
+				: undefined;
+		const resolvedOptions: { preserveRetry?: boolean } | undefined = pending
+			? options
+			: (pendingOrOptions as { preserveRetry?: boolean } | undefined);
+		if (pending && !this.isCurrentRequest(hash, pending)) {
+			return false;
+		}
+		this.restartReservations.delete(hash);
+		this.candidateTokens.delete(hash);
 		const retry = this.retries.get(hash);
 		const session = this.sessions.get(hash);
 		if (session) {
@@ -307,36 +591,57 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 			session.pending = undefined;
 			session.contacted.clear();
 			session.confirmed.clear();
-			if (!options?.preserveRetry || session.retry !== retry) {
+			session.revoked.clear();
+			if (!resolvedOptions?.preserveRetry || session.retry !== retry) {
 				session.retry = undefined;
 			}
 		}
+		if (!pending || this.pendingDeletes.get(hash) === pending) {
+			this.pendingDeletes.delete(hash);
+		}
 		this.requestIPruneSent.delete(hash);
 		this.responseReplicatorSet.delete(hash);
-		if (!options?.preserveRetry) {
+		if (!resolvedOptions?.preserveRetry) {
 			this.clearRetry(hash);
 		}
 		this.deleteSessionIfIdle(hash);
+		return true;
 	}
 
 	cleanupPeer(peer: string) {
-		for (const [hash, peers] of this.requestIPruneSent) {
-			peers.delete(peer);
-			this.sessions.get(hash)?.contacted.delete(peer);
-			if (peers.size === 0) {
-				this.requestIPruneSent.delete(hash);
-				this.deleteSessionIfIdle(hash);
+		const invalidated: CheckedPruneInvalidatedGeneration<T, R>[] = [];
+		// Fence the peer from every active generation, including generations that
+		// have not emitted their first request yet. A removal can begin between
+		// generation creation and message emission.
+		for (const [hash, session] of this.sessions) {
+			const pending = session.pending;
+			if (pending && this.pendingDeletes.get(hash) === pending) {
+				session.revoked.add(peer);
+				const candidate = this.getRestartCandidate(hash);
+				if (
+					candidate &&
+					(session.contacted.has(peer) || candidate.leaders.has(peer))
+				) {
+					invalidated.push({
+						...candidate,
+						pending,
+					});
+				}
+			}
+		}
+
+		for (const [hash, peers] of [...this.requestIPruneSent]) {
+			if (peers.has(peer)) {
+				this.removeRequestSent(hash, peer);
 			}
 		}
 
-		for (const [hash, peers] of this.responseReplicatorSet) {
-			peers.delete(peer);
-			this.sessions.get(hash)?.confirmed.delete(peer);
-			if (peers.size === 0) {
-				this.responseReplicatorSet.delete(hash);
-				this.deleteSessionIfIdle(hash);
+		for (const [hash, peers] of [...this.responseReplicatorSet]) {
+			if (peers.has(peer)) {
+				this.removeConfirmedReplicator(hash, peer);
 			}
 		}
+		return invalidated;
 	}
 
 	close() {
@@ -354,5 +659,9 @@ export class CheckedPruneCoordinator<T, R extends "u32" | "u64"> {
 		this.responseReplicatorSet.clear();
 		this.retries.clear();
 		this.sessions.clear();
+		this.grantSends.clear();
+		this.peerRemovalFences.clear();
+		this.restartReservations.clear();
+		this.candidateTokens.clear();
 	}
 }

package/src/exchange-heads.ts

@@ -1106,6 +1106,58 @@ export class ResponseIPrune extends TransportMessage {
 	}
 }
 
+@variant(0)
+export class CheckedPruneRequest {
+	@field({ type: "string" })
+	hash: string;
+
+	@field({ type: fixedArray("u8", 32) })
+	requestId: Uint8Array;
+
+	constructor(props: { hash: string; requestId: Uint8Array }) {
+		this.hash = props.hash;
+		this.requestId = props.requestId;
+	}
+}
+
+@variant([0, 11])
+export class RequestIPruneV2 extends TransportMessage {
+	@field({ type: vec(CheckedPruneRequest) })
+	requests: CheckedPruneRequest[];
+
+	constructor(props: {
+		requests: Array<
+			CheckedPruneRequest | { hash: string; requestId: Uint8Array }
+		>;
+	}) {
+		super();
+		this.requests = props.requests.map((request) =>
+			request instanceof CheckedPruneRequest
+				? request
+				: new CheckedPruneRequest(request),
+		);
+	}
+}
+
+@variant([0, 12])
+export class ResponseIPruneV2 extends TransportMessage {
+	@field({ type: vec(CheckedPruneRequest) })
+	requests: CheckedPruneRequest[];
+
+	constructor(props: {
+		requests: Array<
+			CheckedPruneRequest | { hash: string; requestId: Uint8Array }
+		>;
+	}) {
+		super();
+		this.requests = props.requests.map((request) =>
+			request instanceof CheckedPruneRequest
+				? request
+				: new CheckedPruneRequest(request),
+		);
+	}
+}
+
 const MAX_EXCHANGE_MESSAGE_SIZE = 1e5; // 100kb. Too large size might not be faster (even if we can do 5mb)
 export const MAX_RAW_EXCHANGE_MESSAGE_SIZE = 512 * 1024;
 export const EXCHANGE_HEADS_RESOLVE_BATCH_SIZE = 256;
@@ -1204,21 +1256,19 @@ export const createExchangeHeadsMessages = async function* (
 			}
 
 			// Fallback for logs without native reference rows.
-			const refs = (await allEntriesWithUniqueGids(log, entry)).filter((x) => {
-				if (visitedHeads.has(x.hash)) {
-					return false;
-				}
-				visitedHeads.add(x.hash);
-				return true;
-			});
+			const gidRefrences = await collectFallbackUniqueReferenceGids(
+				log,
+				entry,
+				visitedHeads,
+			);
 
-			if (refs.length > 1000) {
-				warn("Large refs count: ", refs.length);
+			if (gidRefrences.length > 1000) {
+				warn("Large refs count: ", gidRefrences.length);
 			}
 			current.push(
 				new EntryWithRefs({
 					entry,
-					gidRefrences: refs.map((x) => x.meta.gid),
+					gidRefrences,
 				}),
 			);
 
@@ -2096,18 +2146,18 @@ const resolveExchangeHeadEntries = async (
 	return resolved;
 };
 
-export const allEntriesWithUniqueGids = async (
+const collectFallbackUniqueReferenceGids = async (
 	log: Log<any>,
 	entry: Entry<any>,
-): Promise<Entry<any>[]> => {
-	// TODO optimize this
-	const map: Map<string, ShallowEntry | Entry<any>> = new Map();
+	visitedHeads: Set<string>,
+): Promise<string[]> => {
+	const hashByGid = new Map<string, string>();
 	let curr: (Entry<any> | ShallowEntry)[] = [entry];
 	while (curr.length > 0) {
 		const nexts: (Entry<any> | ShallowEntry)[] = [];
 		for (const element of curr) {
-			if (!map.has(element.meta.gid)) {
-				map.set(element.meta.gid, element);
+			if (!hashByGid.has(element.meta.gid)) {
+				hashByGid.set(element.meta.gid, element.hash);
 				if (element.meta.type === EntryType.APPEND) {
 					for (const next of element.meta.next) {
 						const indexedEntry = await log.entryIndex.getShallow(next);
@@ -2122,30 +2172,17 @@ export const allEntriesWithUniqueGids = async (
 					}
 				}
 			}
-			curr = nexts;
 		}
+		curr = nexts;
 	}
-	const values = [...map.values()];
-	const resolved: Array<Entry<any> | undefined> = new Array(values.length);
-	const unresolvedHashes: string[] = [];
-	const unresolvedPositions: number[] = [];
-	for (let i = 0; i < values.length; i++) {
-		const value = values[i]!;
-		if (value instanceof Entry) {
-			resolved[i] = value;
+
+	const gidRefrences: string[] = [];
+	for (const [gid, hash] of hashByGid) {
+		if (visitedHeads.has(hash)) {
 			continue;
 		}
-		unresolvedHashes.push(value.hash);
-		unresolvedPositions.push(i);
-	}
-	if (unresolvedHashes.length > 0) {
-		const entries = await log.entryIndex.getMany(unresolvedHashes, {
-			type: "full",
-			ignoreMissing: true,
-		});
-		for (let i = 0; i < entries.length; i++) {
-			resolved[unresolvedPositions[i]!] = entries[i];
-		}
+		visitedHeads.add(hash);
+		gidRefrences.push(gid);
 	}
-	return resolved.filter((x) => !!x) as Entry<any>[];
+	return gidRefrences;
 };