@peerbit/shared-log 11.0.4 → 11.0.5-d8e5bfb

package/src/index.ts

@@ -48,6 +48,7 @@ import {
 } from "@peerbit/stream-interface";
 import {
 	AbortError,
+	TimeoutError,
 	debounceAccumulator,
 	debounceFixedInterval,
 	waitFor,
@@ -1793,6 +1794,9 @@ export class SharedLog<
 
 		await this.rpc.subscribe();
 
+		// mark all our replicaiton ranges as "new", this would allow other peers to understand that we recently reopend our database and might need some sync and warmup
+		await this.updateTimestampOfOwnedReplicationRanges(); // TODO do we need to do this before subscribing?
+
 		// if we had a previous session with replication info, and new replication info dictates that we unreplicate
 		// we should do that. Otherwise if options is a unreplication we dont need to do anything because
 		// we are already unreplicated (as we are just opening)
@@ -1820,6 +1824,38 @@ export class SharedLog<
 		}, RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL);
 	}
 
+	private async updateTimestampOfOwnedReplicationRanges(
+		timestamp: number = +new Date(),
+	) {
+		const all = await this.replicationIndex
+			.iterate({
+				query: { hash: this.node.identity.publicKey.hashcode() },
+			})
+			.all();
+		let bnTimestamp = BigInt(timestamp);
+		for (const x of all) {
+			x.value.timestamp = bnTimestamp;
+			await this.replicationIndex.put(x.value);
+		}
+
+		if (all.length > 0) {
+			// emit mature event
+			const maturityTimeout = setTimeout(
+				() => {
+					this.events.dispatchEvent(
+						new CustomEvent<ReplicationChangeEvent>("replicator:mature", {
+							detail: { publicKey: this.node.identity.publicKey },
+						}),
+					);
+				},
+				await this.getDefaultMinRoleAge(),
+			);
+			this._closeController.signal.addEventListener("abort", () => {
+				clearTimeout(maturityTimeout);
+			});
+		}
+	}
+
 	async afterOpen(): Promise<void> {
 		await super.afterOpen();
 
@@ -2615,12 +2651,16 @@ export class SharedLog<
 		start?: NumberFromType<R>;
 		/** Optional: end of the content range (exclusive) */
 		end?: NumberFromType<R>;
+
+		/** Optional: roleAge (in ms) */
+		roleAge?: number;
 	}) {
 		return calculateCoverage({
 			numbers: this.indexableDomain.numbers,
 			peers: this.replicationIndex,
 			end: properties?.end,
 			start: properties?.start,
+			roleAge: properties?.roleAge,
 		});
 	}
 
@@ -2720,38 +2760,6 @@ export class SharedLog<
 		return set;
 	}
 
-	async waitForReplicator(...keys: PublicSignKey[]) {
-		const check = async () => {
-			for (const k of keys) {
-				const iterator = this.replicationIndex?.iterate(
-					{ query: new StringMatch({ key: "hash", value: k.hashcode() }) },
-					{ reference: true },
-				);
-				const rects = await iterator?.next(1);
-				await iterator.close();
-				const rect = rects[0]?.value;
-				if (
-					!rect ||
-					!isMatured(rect, +new Date(), await this.getDefaultMinRoleAge())
-				) {
-					return false;
-				}
-			}
-			return true;
-		};
-
-		// TODO do event based
-		return waitFor(() => check(), {
-			signal: this._closeController.signal,
-		}).catch((e) => {
-			if (e instanceof AbortError) {
-				// ignore error
-				return;
-			}
-			throw e;
-		});
-	}
-
 	async join(
 		entries: (string | Entry<T> | ShallowEntry)[],
 		options?: {
@@ -2886,6 +2894,100 @@ export class SharedLog<
 		}
 	}
 
+	async waitForReplicator(...keys: PublicSignKey[]) {
+		const check = async () => {
+			for (const k of keys) {
+				const iterator = this.replicationIndex?.iterate(
+					{ query: new StringMatch({ key: "hash", value: k.hashcode() }) },
+					{ reference: true },
+				);
+				const rects = await iterator?.next(1);
+				await iterator.close();
+				const rect = rects[0]?.value;
+				if (
+					!rect ||
+					!isMatured(rect, +new Date(), await this.getDefaultMinRoleAge())
+				) {
+					return false;
+				}
+			}
+			return true;
+		};
+
+		// TODO do event based
+		return waitFor(() => check(), {
+			signal: this._closeController.signal,
+		}).catch((e) => {
+			if (e instanceof AbortError) {
+				// ignore error
+				return;
+			}
+			throw e;
+		});
+	}
+
+	async waitForReplicators(options?: {
+		timeout?: number;
+		roleAge?: number;
+		signal?: AbortSignal;
+		coverageThreshold?: number;
+	}) {
+		let coverageThreshold = options?.coverageThreshold ?? 0.99;
+		let deferred = pDefer<void>();
+		const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge());
+		const providedCustomRoleAge = options?.roleAge != null;
+
+		let checkCoverage = async () => {
+			const coverage = await this.calculateCoverage({
+				roleAge,
+			});
+			if (coverage > coverageThreshold) {
+				deferred.resolve();
+				return true;
+			}
+			return false;
+		};
+		this.events.addEventListener("replicator:mature", checkCoverage);
+		this.events.addEventListener("replication:change", checkCoverage);
+		await checkCoverage();
+
+		let interval = providedCustomRoleAge
+			? setInterval(() => {
+					checkCoverage();
+				}, 100)
+			: undefined;
+
+		let timeout = options?.timeout ?? this.waitForReplicatorTimeout;
+		const timer = setTimeout(() => {
+			clear();
+			deferred.reject(
+				new TimeoutError(`Timeout waiting for mature replicators`),
+			);
+		}, timeout);
+
+		const abortListener = () => {
+			clear();
+			deferred.reject(new AbortError());
+		};
+
+		if (options?.signal) {
+			options.signal.addEventListener("abort", abortListener);
+		}
+		const clear = () => {
+			interval && clearInterval(interval);
+			this.events.removeEventListener("join", checkCoverage);
+			this.events.removeEventListener("leave", checkCoverage);
+			clearTimeout(timer);
+			if (options?.signal) {
+				options.signal.removeEventListener("abort", abortListener);
+			}
+		};
+
+		return deferred.promise.finally(() => {
+			return clear();
+		});
+	}
+
 	private async _waitForReplicators(
 		cursors: NumberFromType<R>[],
 		entry: Entry<T> | EntryReplicated<R> | ShallowEntry,
@@ -3046,9 +3148,10 @@ export class SharedLog<
 		}
 
 		const now = +new Date();
-		const subscribers =
-			(await this.node.services.pubsub.getSubscribers(this.rpc.topic))
-				?.length ?? 1;
+		const subscribers = this.rpc.closed
+			? 1
+			: ((await this.node.services.pubsub.getSubscribers(this.rpc.topic))
+					?.length ?? 1);
 		const diffToOldest =
 			subscribers > 1 ? now - this.oldestOpenTime - 1 : Number.MAX_SAFE_INTEGER;
 

package/src/ranges.ts

@@ -1372,6 +1372,9 @@ export const calculateCoverage = async <R extends "u32" | "u64">(properties: {
 	start?: NumberFromType<R>;
 	/** Optional: end of the content range (exclusive) */
 	end?: NumberFromType<R>;
+
+	/** Optional: role age limit in milliseconds */
+	roleAge?: number;
 }): Promise<number> => {
 	// Use the provided content range if given; otherwise use the default full range.
 	const contentStart = properties.start ?? properties.numbers.zero;
@@ -1385,12 +1388,14 @@ export const calculateCoverage = async <R extends "u32" | "u64">(properties: {
 			numbers: properties.numbers,
 			start: contentStart,
 			end: properties.numbers.maxValue,
+			roleAge: properties.roleAge,
 		});
 		const coverage2 = await calculateCoverage({
 			peers: properties.peers,
 			numbers: properties.numbers,
 			start: properties.numbers.zero,
 			end: contentEnd,
+			roleAge: properties.roleAge,
 		});
 
 		return Math.min(coverage1, coverage2);
@@ -1399,7 +1404,21 @@ export const calculateCoverage = async <R extends "u32" | "u64">(properties: {
 	const endpoints: { point: NumberFromType<R>; delta: -1 | 1 }[] = [];
 
 	// For each range, record its start and end as events.
-	for (const r of await properties.peers.iterate().all()) {
+	const timeThresholdQuery =
+		properties?.roleAge != null
+			? [
+					new IntegerCompare({
+						key: "timestamp",
+						compare: Compare.LessOrEqual,
+						value: BigInt(Date.now() - properties.roleAge),
+					}),
+				]
+			: undefined;
+	for (const r of await properties.peers
+		.iterate({
+			query: timeThresholdQuery,
+		})
+		.all()) {
 		endpoints.push({ point: r.value.start1, delta: +1 });
 		endpoints.push({ point: r.value.end1, delta: -1 });
 

package/src/sync/rateless-iblt.ts

@@ -541,7 +541,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			if (outProcess === true) {
 				return true;
 			} else if (outProcess === undefined) {
-				return false; // we don't have enough information, or received information that is redundant
+				return true; // we don't have enough information, or received information that is redundant
 			}
 
 			// we are not done