@peerbit/shared-log 12.1.3 → 12.2.0-3333888

package/src/sync/rateless-iblt.ts

@@ -4,18 +4,24 @@ import { type PublicSignKey, randomBytes, toBase64 } from "@peerbit/crypto";
 import { type Index } from "@peerbit/indexer-interface";
 import type { Entry, Log } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
-import { DecoderWrapper, EncoderWrapper } from "@peerbit/riblt";
+import {
+	DecoderWrapper,
+	EncoderWrapper,
+	ready as ribltReady,
+} from "@peerbit/riblt";
 import type { RPC, RequestContext } from "@peerbit/rpc";
 import { SilentDelivery } from "@peerbit/stream-interface";
 import { type EntryWithRefs } from "../exchange-heads.js";
-import { type Numbers } from "../integers.js";
 import { TransportMessage } from "../message.js";
 import {
 	type EntryReplicated,
-	type ReplicationRangeIndexable,
 	matchEntriesInRangeQuery,
 } from "../ranges.js";
-import type { SyncableKey, Syncronizer } from "./index.js";
+import type {
+	SyncableKey,
+	SynchronizerComponents,
+	Syncronizer,
+} from "./index.js";
 import { SimpleSyncronizer } from "./simple.js";
 
 export const logger = loggerFn("peerbit:shared-log:rateless");
@@ -142,6 +148,7 @@ const buildEncoderOrDecoderFromRange = async <
 	entryIndex: Index<EntryReplicated<D>>,
 	type: T,
 ): Promise<E | false> => {
+	await ribltReady;
 	const encoder =
 		type === "encoder" ? new EncoderWrapper() : new DecoderWrapper();
 
@@ -180,6 +187,13 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	simple: SimpleSyncronizer<D>;
 
 	startedOrCompletedSynchronizations: Cache<string>;
+	private localRangeEncoderCacheVersion = 0;
+	private localRangeEncoderCache: Map<
+		string,
+		{ encoder: EncoderWrapper; version: number; lastUsed: number }
+	> = new Map();
+	private localRangeEncoderCacheMax = 2;
+
 	ingoingSyncProcesses: Map<
 		string,
 		{
@@ -207,14 +221,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	>;
 
 	constructor(
-		readonly properties: {
-			rpc: RPC<TransportMessage, TransportMessage>;
-			rangeIndex: Index<ReplicationRangeIndexable<D>, any>;
-			entryIndex: Index<EntryReplicated<D>, any>;
-			log: Log<any>;
-			coordinateToHash: Cache<string>;
-			numbers: Numbers<D>;
-		},
+		readonly properties: SynchronizerComponents<D>,
 	) {
 		this.simple = new SimpleSyncronizer(properties);
 		this.outgoingSyncProcesses = new Map();
@@ -222,6 +229,91 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
 	}
 
+	private clearLocalRangeEncoderCache() {
+		for (const [, cached] of this.localRangeEncoderCache) {
+			cached.encoder.free();
+		}
+		this.localRangeEncoderCache.clear();
+	}
+
+	private invalidateLocalRangeEncoderCache() {
+		this.localRangeEncoderCacheVersion += 1;
+		this.clearLocalRangeEncoderCache();
+	}
+
+	private localRangeEncoderCacheKey(ranges: {
+		start1: NumberOrBigint;
+		end1: NumberOrBigint;
+		start2: NumberOrBigint;
+		end2: NumberOrBigint;
+	}) {
+		return `${String(ranges.start1)}:${String(ranges.end1)}:${String(
+			ranges.start2,
+		)}:${String(ranges.end2)}`;
+	}
+
+	private decoderFromCachedEncoder(encoder: EncoderWrapper): DecoderWrapper {
+		const clone = encoder.clone();
+		const decoder = clone.to_decoder();
+		clone.free();
+		return decoder;
+	}
+
+	private async getLocalDecoderForRange(ranges: {
+		start1: NumberOrBigint;
+		end1: NumberOrBigint;
+		start2: NumberOrBigint;
+		end2: NumberOrBigint;
+	}): Promise<DecoderWrapper | false> {
+		const key = this.localRangeEncoderCacheKey(ranges);
+		const cached = this.localRangeEncoderCache.get(key);
+		if (cached && cached.version === this.localRangeEncoderCacheVersion) {
+			cached.lastUsed = Date.now();
+			return this.decoderFromCachedEncoder(cached.encoder);
+		}
+
+		const encoder = (await buildEncoderOrDecoderFromRange(
+			ranges,
+			this.properties.entryIndex,
+			"encoder",
+		)) as EncoderWrapper | false;
+		if (!encoder) {
+			return false;
+		}
+
+		const now = Date.now();
+		const existing = this.localRangeEncoderCache.get(key);
+		if (existing) {
+			existing.encoder.free();
+		}
+		this.localRangeEncoderCache.set(key, {
+			encoder,
+			version: this.localRangeEncoderCacheVersion,
+			lastUsed: now,
+		});
+
+		while (this.localRangeEncoderCache.size > this.localRangeEncoderCacheMax) {
+			let oldestKey: string | undefined;
+			let oldestUsed = Number.POSITIVE_INFINITY;
+			for (const [candidateKey, value] of this.localRangeEncoderCache) {
+				if (value.lastUsed < oldestUsed) {
+					oldestUsed = value.lastUsed;
+					oldestKey = candidateKey;
+				}
+			}
+			if (!oldestKey) {
+				break;
+			}
+			const victim = this.localRangeEncoderCache.get(oldestKey);
+			if (victim) {
+				victim.encoder.free();
+			}
+			this.localRangeEncoderCache.delete(oldestKey);
+		}
+
+		return this.decoderFromCachedEncoder(encoder);
+	}
+
 	async onMaybeMissingEntries(properties: {
 		entries: Map<string, EntryReplicated<D>>;
 		targets: string[];
@@ -233,7 +325,6 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		//   such as those assigned to range boundaries.
 
 		let entriesToSyncNaively: Map<string, EntryReplicated<D>> = new Map();
-		let allCoordinatesToSyncWithIblt: bigint[] = [];
 		let minSyncIbltSize = 333; // TODO: make configurable
 		let maxSyncWithSimpleMethod = 1e3;
 
@@ -246,13 +337,59 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			return;
 		}
 
-		// Mixed strategy for larger batches
+		const nonBoundaryEntries: EntryReplicated<D>[] = [];
 		for (const entry of properties.entries.values()) {
 			if (entry.assignedToRangeBoundary) {
 				entriesToSyncNaively.set(entry.hash, entry);
 			} else {
-				allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
+				nonBoundaryEntries.push(entry);
+			}
+		}
+
+		const priorityFn = this.properties.sync?.priority;
+		const maxSimpleEntries = this.properties.sync?.maxSimpleEntries;
+		const maxAdditionalNaive =
+			priorityFn &&
+			typeof maxSimpleEntries === "number" &&
+			Number.isFinite(maxSimpleEntries) &&
+			maxSimpleEntries > 0
+				? Math.max(
+						0,
+						Math.min(
+							Math.floor(maxSimpleEntries),
+							maxSyncWithSimpleMethod - entriesToSyncNaively.size,
+						),
+					)
+				: 0;
+
+		if (priorityFn && maxAdditionalNaive > 0 && nonBoundaryEntries.length > 0) {
+			let index = 0;
+			const scored: {
+				entry: EntryReplicated<D>;
+				index: number;
+				priority: number;
+			}[] = [];
+			for (const entry of nonBoundaryEntries) {
+				const priorityValue = priorityFn(entry);
+				scored.push({
+					entry,
+					index,
+					priority: Number.isFinite(priorityValue) ? priorityValue : 0,
+				});
+				index += 1;
+			}
+			scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
+			for (const { entry } of scored.slice(0, maxAdditionalNaive)) {
+				entriesToSyncNaively.set(entry.hash, entry);
+			}
+		}
+
+		let allCoordinatesToSyncWithIblt: bigint[] = [];
+		for (const entry of nonBoundaryEntries) {
+			if (entriesToSyncNaively.has(entry.hash)) {
+				continue;
 			}
+			allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
 		}
 
 		if (entriesToSyncNaively.size > 0) {
@@ -268,31 +405,44 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			entriesToSyncNaively.size > maxSyncWithSimpleMethod
 		) {
 			// Fallback: if nothing left for IBLT (or simple set is too large), include all in IBLT
-			allCoordinatesToSyncWithIblt = Array.from(
-				properties.entries.values(),
-			).map((x) => coerceBigInt(x.hashNumber));
+			allCoordinatesToSyncWithIblt = [];
+			for (const entry of properties.entries.values()) {
+				allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
+			}
 		}
 
 		if (allCoordinatesToSyncWithIblt.length === 0) {
 			return;
 		}
 
-		const sortedEntries = allCoordinatesToSyncWithIblt.sort((a, b) => {
-			if (a > b) {
-				return 1;
-			} else if (a < b) {
-				return -1;
-			} else {
-				return 0;
+		await ribltReady;
+
+		let sortedEntries: bigint[] | BigUint64Array;
+		if (typeof BigUint64Array !== "undefined") {
+			const typed = new BigUint64Array(allCoordinatesToSyncWithIblt.length);
+			for (let i = 0; i < allCoordinatesToSyncWithIblt.length; i++) {
+				typed[i] = allCoordinatesToSyncWithIblt[i];
 			}
-		});
+			typed.sort();
+			sortedEntries = typed;
+		} else {
+			sortedEntries = allCoordinatesToSyncWithIblt.sort((a, b) => {
+				if (a > b) {
+					return 1;
+				} else if (a < b) {
+					return -1;
+				} else {
+					return 0;
+				}
+			});
+		}
 
 		// assume sorted, and find the largest gap
 		let largestGap = 0n;
 		let largestGapIndex = 0;
-		for (let i = 0; i < sortedEntries.length - 1; i++) {
+		for (let i = 0; i < sortedEntries.length; i++) {
 			const current = sortedEntries[i];
-			const next = sortedEntries[i + 1];
+			const next = sortedEntries[(i + 1) % sortedEntries.length];
 			const gap =
 				next >= current
 					? next - current
@@ -322,8 +472,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 
 		const startSync = new StartSync({ from: start, to: end, symbols: [] });
 		const encoder = new EncoderWrapper();
-		for (const entry of sortedEntries) {
-			encoder.add_symbol(coerceBigInt(entry));
+		if (typeof BigUint64Array !== "undefined" && sortedEntries instanceof BigUint64Array) {
+			encoder.add_symbols(sortedEntries);
+		} else {
+			for (const entry of sortedEntries) {
+				encoder.add_symbol(coerceBigInt(entry));
+			}
 		}
 
 		let initialSymbols = Math.round(
@@ -399,16 +553,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			this.startedOrCompletedSynchronizations.add(syncId);
 
 			const wrapped = message.end < message.start;
-			const decoder = await buildEncoderOrDecoderFromRange(
-				{
-					start1: message.start,
-					end1: wrapped ? this.properties.numbers.maxValue : message.end,
-					start2: 0n,
-					end2: wrapped ? message.end : 0n,
-				},
-				this.properties.entryIndex,
-				"decoder",
-			);
+			const decoder = await this.getLocalDecoderForRange({
+				start1: message.start,
+				end1: wrapped ? this.properties.numbers.maxValue : message.end,
+				start2: 0n,
+				end2: wrapped ? message.end : 0n,
+			});
 
 			if (!decoder) {
 				await this.simple.rpc.send(
@@ -613,10 +763,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	}
 
 	onEntryAdded(entry: Entry<any>): void {
+		this.invalidateLocalRangeEncoderCache();
 		return this.simple.onEntryAdded(entry);
 	}
 
 	onEntryRemoved(hash: string) {
+		this.invalidateLocalRangeEncoderCache();
 		return this.simple.onEntryRemoved(hash);
 	}
 
@@ -635,6 +787,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		for (const [, obj] of this.outgoingSyncProcesses) {
 			obj.free();
 		}
+		this.clearLocalRangeEncoderCache();
 		return this.simple.close();
 	}
 

package/src/sync/simple.ts

@@ -16,7 +16,7 @@ import {
 } from "../exchange-heads.js";
 import { TransportMessage } from "../message.js";
 import type { EntryReplicated } from "../ranges.js";
-import type { SyncableKey, Syncronizer } from "./index.js";
+import type { SyncableKey, SyncOptions, Syncronizer } from "./index.js";
 
 @variant([0, 1])
 export class RequestMaybeSync extends TransportMessage {
@@ -57,7 +57,7 @@ const getHashesFromSymbols = async (
 	coordinateToHash: Cache<string>,
 ) => {
 	let queries: IntegerCompare[] = [];
-	let batchSize = 1; // TODO arg
+	let batchSize = 128; // TODO arg
 	let results = new Set<string>();
 	const handleBatch = async (end = false) => {
 		if (queries.length >= batchSize || (end && queries.length > 0)) {
@@ -109,6 +109,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 	log: Log<any>;
 	entryIndex: Index<EntryReplicated<R>, any>;
 	coordinateToHash: Cache<string>;
+	private syncOptions?: SyncOptions<R>;
 
 	// Syncing and dedeplucation work
 	syncMoreInterval?: ReturnType<typeof setTimeout>;
@@ -120,6 +121,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 		entryIndex: Index<EntryReplicated<R>, any>;
 		log: Log<any>;
 		coordinateToHash: Cache<string>;
+		sync?: SyncOptions<R>;
 	}) {
 		this.syncInFlightQueue = new Map();
 		this.syncInFlightQueueInverted = new Map();
@@ -128,14 +130,35 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 		this.log = properties.log;
 		this.entryIndex = properties.entryIndex;
 		this.coordinateToHash = properties.coordinateToHash;
+		this.syncOptions = properties.sync;
 	}
 
 	onMaybeMissingEntries(properties: {
 		entries: Map<string, EntryReplicated<R>>;
 		targets: string[];
 	}): Promise<void> {
+		let hashes: string[];
+		const priorityFn = this.syncOptions?.priority;
+		if (priorityFn) {
+			let index = 0;
+			const scored: { hash: string; index: number; priority: number }[] = [];
+			for (const [hash, entry] of properties.entries) {
+				const priorityValue = priorityFn(entry);
+				scored.push({
+					hash,
+					index,
+					priority: Number.isFinite(priorityValue) ? priorityValue : 0,
+				});
+				index += 1;
+			}
+			scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
+			hashes = scored.map((x) => x.hash);
+		} else {
+			hashes = [...properties.entries.keys()];
+		}
+
 		return this.rpc.send(
-			new RequestMaybeSync({ hashes: [...properties.entries.keys()] }),
+			new RequestMaybeSync({ hashes }),
 			{
 				priority: 1,
 				mode: new SilentDelivery({ to: properties.targets, redundancy: 1 }),