@peerbit/shared-log 12.2.0-874976b → 12.2.0-90d77b6

package/src/ranges.ts

@@ -2064,6 +2064,7 @@ export const getSamples = async <R extends "u32" | "u64">(
 	options?: {
 		onlyIntersecting?: boolean;
 		uniqueReplicators?: Set<string>;
+		peerFilter?: Set<string>;
 	},
 ): Promise<Map<string, { intersecting: boolean }>> => {
 	const leaders: Map<string, { intersecting: boolean }> = new Map();
@@ -2075,6 +2076,7 @@ export const getSamples = async <R extends "u32" | "u64">(
 	let matured = 0;
 
 	let uniqueVisited = new Set<string>();
+	const peerFilter = options?.peerFilter;
 	for (let i = 0; i < cursor.length; i++) {
 		let point = cursor[i];
 
@@ -2084,6 +2086,9 @@ export const getSamples = async <R extends "u32" | "u64">(
 		);
 
 		for (const rect of allContaining) {
+			if (peerFilter && !peerFilter.has(rect.value.hash)) {
+				continue;
+			}
 			uniqueVisited.add(rect.value.hash);
 			let prev = leaders.get(rect.value.hash);
 			if (!prev) {
@@ -2096,7 +2101,7 @@ export const getSamples = async <R extends "u32" | "u64">(
 			}
 		}
 
-		if (options?.uniqueReplicators) {
+		if (options?.uniqueReplicators && options.uniqueReplicators.size > 0) {
 			if (
 				options.uniqueReplicators.size === leaders.size ||
 				options.uniqueReplicators.size === uniqueVisited.size
@@ -2114,6 +2119,9 @@ export const getSamples = async <R extends "u32" | "u64">(
 			roleAge,
 			peers,
 			(rect, m) => {
+				if (peerFilter && !peerFilter.has(rect.hash)) {
+					return;
+				}
 				uniqueVisited.add(rect.hash);
 				const prev = leaders.get(rect.hash);
 				if (m) {
@@ -2234,7 +2242,7 @@ export const getCoverSet = async <R extends "u32" | "u64">(properties: {
 		nextLocation: NumberFromType<R>,
 		roleAge: number,
 	) => {
-		let next = await fetchOne(
+		const next = await fetchOne(
 			iterateRangesContainingPoint<undefined, R>(peers, nextLocation, {
 				sort: [new Sort({ key: "end2", direction: SortDirection.DESC })],
 				time: {
@@ -2243,31 +2251,31 @@ export const getCoverSet = async <R extends "u32" | "u64">(properties: {
 					now,
 				},
 			}),
-		); // get entersecting sort by largest end2
+		); // get intersecting sort by largest end2
 		return next;
 	};
 
-	const resolveNextAbove = async (
-		nextLocation: NumberFromType<R>,
-		roleAge: number,
-	) => {
-		// if not get closest from above
-		let next = await fetchOne<undefined, R>(
-			getClosest("above", peers, nextLocation, true, properties.numbers, {
-				time: {
-					matured: true,
-					roleAgeLimit: roleAge,
-					now,
-				},
-			}),
-		);
-		return next;
-	};
+		const resolveNextAbove = async (
+			nextLocation: NumberFromType<R>,
+			roleAge: number,
+		) => {
+			// if not get closest from above
+			const next = await fetchOne<undefined, R>(
+				getClosest("above", peers, nextLocation, true, properties.numbers, {
+					time: {
+						matured: true,
+						roleAgeLimit: roleAge,
+						now,
+					},
+				}),
+			);
+			return next;
+		};
 
 	const resolveNext = async (
 		nextLocation: NumberFromType<R>,
 		roleAge: number,
-	): Promise<[ReplicationRangeIndexable<R>, boolean]> => {
+	): Promise<[ReplicationRangeIndexable<R> | undefined, boolean]> => {
 		const containing = await resolveNextContaining(nextLocation, roleAge);
 		if (containing) {
 			return [containing, true];
@@ -2288,12 +2296,25 @@ export const getCoverSet = async <R extends "u32" | "u64">(properties: {
 		from: NumberFromType<R>,
 	) => {
 		const toEnd2 = properties.numbers.increment(to.end2); // TODO investigate why this is needed
-		if (toEnd2 < from || to.wrapped) {
+		if (toEnd2 < from) {
 			wrappedOnce = true;
 			// @ts-ignore
 			coveredLength += properties.numbers.maxValue - from;
 			// @ts-ignore
 			coveredLength += toEnd2;
+		} else if (to.wrapped) {
+			// When the range is wrapped and `from` is in the second segment (near zero),
+			// the distance to `end2` does not wrap. Otherwise we must wrap to reach `end2`.
+			if (from < to.end2) {
+				// @ts-ignore
+				coveredLength += toEnd2 - from;
+			} else {
+				wrappedOnce = true;
+				// @ts-ignore
+				coveredLength += properties.numbers.maxValue - from;
+				// @ts-ignore
+				coveredLength += toEnd2;
+			}
 		} else {
 			// @ts-ignore
 			coveredLength += to.end1 - from;
@@ -2314,6 +2335,7 @@ export const getCoverSet = async <R extends "u32" | "u64">(properties: {
 		(coveredLength <= properties.numbers.maxValue || !wrappedOnce) // eslint-disable-line no-unmodified-loop-condition
 	) {
 		let distanceBefore = coveredLength;
+		const nextLocationBefore = nextLocation;
 
 		let nextCandidate = await resolveNext(nextLocation, roleAge);
 		let matured = true;
@@ -2343,55 +2365,53 @@ export const getCoverSet = async <R extends "u32" | "u64">(properties: {
 		let last = current;
 		current = nextCandidate[0];
 
-		let isLast =
-			distanceBefore < widthToCoverScaled &&
-			coveredLength >= widthToCoverScaled;
+		const isLast =
+			distanceBefore < widthToCoverScaled && coveredLength >= widthToCoverScaled;
+
+		const lastDistanceToEndLocation = properties.numbers.min(
+			getDistance(
+				last.start1,
+				endLocation,
+				"closest",
+				properties.numbers.maxValue,
+			),
+			getDistance(
+				last.end2,
+				endLocation,
+				"closest",
+				properties.numbers.maxValue,
+			),
+		);
+
+		const currentDistanceToEndLocation = properties.numbers.min(
+			getDistance(
+				current.start1,
+				endLocation,
+				"closest",
+				properties.numbers.maxValue,
+			),
+			getDistance(
+				current.end2,
+				endLocation,
+				"closest",
+				properties.numbers.maxValue,
+			),
+		);
 
 		if (
 			!isLast ||
 			nextCandidate[1] ||
-			properties.numbers.min(
-				getDistance(
-					last.start1,
-					endLocation,
-					"closest",
-					properties.numbers.maxValue,
-				),
-				getDistance(
-					last.end2,
-					endLocation,
-					"closest",
-					properties.numbers.maxValue,
-				),
-			) >
-				properties.numbers.min(
-					getDistance(
-						current.start1,
-						endLocation,
-						"closest",
-						properties.numbers.maxValue,
-					),
-					getDistance(
-						current.end2,
-						endLocation,
-						"closest",
-						properties.numbers.maxValue,
-					),
-				)
+			lastDistanceToEndLocation >= currentDistanceToEndLocation
 		) {
 			ret.add(current.hash);
 		}
 
-		if (isLast && !nextCandidate[1] /*  || equals(endRect.id, current.id) */) {
-			break;
-		}
-
 		if (matured) {
 			maturedCoveredLength = coveredLength;
 		}
 
 		let startForNext = extraDistanceForNext
-			? properties.numbers.increment(current.end2)
+			? properties.numbers.increment(nextLocation)
 			: current.end2;
 		nextLocation = endIsWrapped
 			? wrappedOnce
@@ -2399,6 +2419,15 @@ export const getCoverSet = async <R extends "u32" | "u64">(properties: {
 				: startForNext
 			: properties.numbers.min(startForNext, endLocation);
 
+		// Safety: ensure we always make progress to avoid infinite loops (can happen when
+		// the chosen range is the same and `nextLocation` doesn't advance).
+		if (
+			nextLocation === nextLocationBefore &&
+			coveredLength === distanceBefore
+		) {
+			break;
+		}
+
 		if (
 			(typeof nextLocation === "bigint" &&
 				nextLocation === (endLocation as bigint)) ||
@@ -2603,19 +2632,19 @@ export const mergeReplicationChanges = <R extends NumericType>(
 			let results: ReplicationChange<ReplicationRangeIndexable<R>>[] = [];
 			let consumed: Set<number> = new Set();
 			for (let i = 0; i < v.length; i++) {
-				// if segment is removed and we have previously processed it
-				// then go over each overlapping added segment add remove the removal,
-				// equivalent is that this would represent  (1 - 1 + 1) = 1
+				// If segment is removed and we have previously processed it then go over each
+				// overlapping added segment and remove the overlap. Equivalent to: (1 - 1 + 1) = 1.
 				if (v[i].type === "removed" || v[i].type === "replaced") {
 					if (rebalanceHistory.has(v[i].range.rangeHash)) {
-						let vStart = v.length;
+						let adjusted = false;
+						const vStart = v.length;
 						for (let j = i + 1; j < vStart; j++) {
 							const newer = v[j];
 							if (newer.type === "added" && !newer.matured) {
-								const {
-									rangesFromA: updatedRemoved,
-									rangesFromB: updatedNewer,
-								} = symmetricDifferenceRanges(v[i].range, newer.range);
+								adjusted = true;
+								const { rangesFromA: updatedRemoved, rangesFromB: updatedNewer } =
+									symmetricDifferenceRanges(v[i].range, newer.range);
+
 								for (const diff of updatedRemoved) {
 									results.push({
 										range: diff,
@@ -2634,6 +2663,9 @@ export const mergeReplicationChanges = <R extends NumericType>(
 							}
 						}
 						rebalanceHistory.del(v[i].range.rangeHash);
+						if (!adjusted) {
+							results.push(v[i]);
+						}
 					} else {
 						results.push(v[i]);
 					}

package/src/sync/rateless-iblt.ts

@@ -187,6 +187,13 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	simple: SimpleSyncronizer<D>;
 
 	startedOrCompletedSynchronizations: Cache<string>;
+	private localRangeEncoderCacheVersion = 0;
+	private localRangeEncoderCache: Map<
+		string,
+		{ encoder: EncoderWrapper; version: number; lastUsed: number }
+	> = new Map();
+	private localRangeEncoderCacheMax = 2;
+
 	ingoingSyncProcesses: Map<
 		string,
 		{
@@ -222,6 +229,91 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
 	}
 
+	private clearLocalRangeEncoderCache() {
+		for (const [, cached] of this.localRangeEncoderCache) {
+			cached.encoder.free();
+		}
+		this.localRangeEncoderCache.clear();
+	}
+
+	private invalidateLocalRangeEncoderCache() {
+		this.localRangeEncoderCacheVersion += 1;
+		this.clearLocalRangeEncoderCache();
+	}
+
+	private localRangeEncoderCacheKey(ranges: {
+		start1: NumberOrBigint;
+		end1: NumberOrBigint;
+		start2: NumberOrBigint;
+		end2: NumberOrBigint;
+	}) {
+		return `${String(ranges.start1)}:${String(ranges.end1)}:${String(
+			ranges.start2,
+		)}:${String(ranges.end2)}`;
+	}
+
+	private decoderFromCachedEncoder(encoder: EncoderWrapper): DecoderWrapper {
+		const clone = encoder.clone();
+		const decoder = clone.to_decoder();
+		clone.free();
+		return decoder;
+	}
+
+	private async getLocalDecoderForRange(ranges: {
+		start1: NumberOrBigint;
+		end1: NumberOrBigint;
+		start2: NumberOrBigint;
+		end2: NumberOrBigint;
+	}): Promise<DecoderWrapper | false> {
+		const key = this.localRangeEncoderCacheKey(ranges);
+		const cached = this.localRangeEncoderCache.get(key);
+		if (cached && cached.version === this.localRangeEncoderCacheVersion) {
+			cached.lastUsed = Date.now();
+			return this.decoderFromCachedEncoder(cached.encoder);
+		}
+
+		const encoder = (await buildEncoderOrDecoderFromRange(
+			ranges,
+			this.properties.entryIndex,
+			"encoder",
+		)) as EncoderWrapper | false;
+		if (!encoder) {
+			return false;
+		}
+
+		const now = Date.now();
+		const existing = this.localRangeEncoderCache.get(key);
+		if (existing) {
+			existing.encoder.free();
+		}
+		this.localRangeEncoderCache.set(key, {
+			encoder,
+			version: this.localRangeEncoderCacheVersion,
+			lastUsed: now,
+		});
+
+		while (this.localRangeEncoderCache.size > this.localRangeEncoderCacheMax) {
+			let oldestKey: string | undefined;
+			let oldestUsed = Number.POSITIVE_INFINITY;
+			for (const [candidateKey, value] of this.localRangeEncoderCache) {
+				if (value.lastUsed < oldestUsed) {
+					oldestUsed = value.lastUsed;
+					oldestKey = candidateKey;
+				}
+			}
+			if (!oldestKey) {
+				break;
+			}
+			const victim = this.localRangeEncoderCache.get(oldestKey);
+			if (victim) {
+				victim.encoder.free();
+			}
+			this.localRangeEncoderCache.delete(oldestKey);
+		}
+
+		return this.decoderFromCachedEncoder(encoder);
+	}
+
 	async onMaybeMissingEntries(properties: {
 		entries: Map<string, EntryReplicated<D>>;
 		targets: string[];
@@ -292,9 +384,13 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			}
 		}
 
-		let allCoordinatesToSyncWithIblt = nonBoundaryEntries
-			.filter((entry) => !entriesToSyncNaively.has(entry.hash))
-			.map((entry) => coerceBigInt(entry.hashNumber));
+		let allCoordinatesToSyncWithIblt: bigint[] = [];
+		for (const entry of nonBoundaryEntries) {
+			if (entriesToSyncNaively.has(entry.hash)) {
+				continue;
+			}
+			allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
+		}
 
 		if (entriesToSyncNaively.size > 0) {
 			// If there are special-case entries, sync them simply in parallel
@@ -309,9 +405,10 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			entriesToSyncNaively.size > maxSyncWithSimpleMethod
 		) {
 			// Fallback: if nothing left for IBLT (or simple set is too large), include all in IBLT
-			allCoordinatesToSyncWithIblt = Array.from(
-				properties.entries.values(),
-			).map((x) => coerceBigInt(x.hashNumber));
+			allCoordinatesToSyncWithIblt = [];
+			for (const entry of properties.entries.values()) {
+				allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
+			}
 		}
 
 		if (allCoordinatesToSyncWithIblt.length === 0) {
@@ -320,22 +417,32 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 
 		await ribltReady;
 
-		const sortedEntries = allCoordinatesToSyncWithIblt.sort((a, b) => {
-			if (a > b) {
-				return 1;
-			} else if (a < b) {
-				return -1;
-			} else {
-				return 0;
+		let sortedEntries: bigint[] | BigUint64Array;
+		if (typeof BigUint64Array !== "undefined") {
+			const typed = new BigUint64Array(allCoordinatesToSyncWithIblt.length);
+			for (let i = 0; i < allCoordinatesToSyncWithIblt.length; i++) {
+				typed[i] = allCoordinatesToSyncWithIblt[i];
 			}
-		});
+			typed.sort();
+			sortedEntries = typed;
+		} else {
+			sortedEntries = allCoordinatesToSyncWithIblt.sort((a, b) => {
+				if (a > b) {
+					return 1;
+				} else if (a < b) {
+					return -1;
+				} else {
+					return 0;
+				}
+			});
+		}
 
 		// assume sorted, and find the largest gap
 		let largestGap = 0n;
 		let largestGapIndex = 0;
-		for (let i = 0; i < sortedEntries.length - 1; i++) {
+		for (let i = 0; i < sortedEntries.length; i++) {
 			const current = sortedEntries[i];
-			const next = sortedEntries[i + 1];
+			const next = sortedEntries[(i + 1) % sortedEntries.length];
 			const gap =
 				next >= current
 					? next - current
@@ -365,8 +472,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 
 		const startSync = new StartSync({ from: start, to: end, symbols: [] });
 		const encoder = new EncoderWrapper();
-		for (const entry of sortedEntries) {
-			encoder.add_symbol(coerceBigInt(entry));
+		if (typeof BigUint64Array !== "undefined" && sortedEntries instanceof BigUint64Array) {
+			encoder.add_symbols(sortedEntries);
+		} else {
+			for (const entry of sortedEntries) {
+				encoder.add_symbol(coerceBigInt(entry));
+			}
 		}
 
 		let initialSymbols = Math.round(
@@ -442,16 +553,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			this.startedOrCompletedSynchronizations.add(syncId);
 
 			const wrapped = message.end < message.start;
-			const decoder = await buildEncoderOrDecoderFromRange(
-				{
-					start1: message.start,
-					end1: wrapped ? this.properties.numbers.maxValue : message.end,
-					start2: 0n,
-					end2: wrapped ? message.end : 0n,
-				},
-				this.properties.entryIndex,
-				"decoder",
-			);
+			const decoder = await this.getLocalDecoderForRange({
+				start1: message.start,
+				end1: wrapped ? this.properties.numbers.maxValue : message.end,
+				start2: 0n,
+				end2: wrapped ? message.end : 0n,
+			});
 
 			if (!decoder) {
 				await this.simple.rpc.send(
@@ -656,10 +763,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	}
 
 	onEntryAdded(entry: Entry<any>): void {
+		this.invalidateLocalRangeEncoderCache();
 		return this.simple.onEntryAdded(entry);
 	}
 
 	onEntryRemoved(hash: string) {
+		this.invalidateLocalRangeEncoderCache();
 		return this.simple.onEntryRemoved(hash);
 	}
 
@@ -678,6 +787,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		for (const [, obj] of this.outgoingSyncProcesses) {
 			obj.free();
 		}
+		this.clearLocalRangeEncoderCache();
 		return this.simple.close();
 	}