@peerbit/shared-log 12.1.3 → 12.2.0-62829ef

package/src/index.ts

@@ -135,7 +135,11 @@ import {
 	maxReplicas,
 } from "./replication.js";
 import { Observer, Replicator } from "./role.js";
-import type { SynchronizerConstructor, Syncronizer } from "./sync/index.js";
+import type {
+	SyncOptions,
+	SynchronizerConstructor,
+	Syncronizer,
+} from "./sync/index.js";
 import { RatelessIBLTSynchronizer } from "./sync/rateless-iblt.js";
 import { SimpleSyncronizer } from "./sync/simple.js";
 import { groupByGid } from "./utils.js";
@@ -149,6 +153,12 @@ export {
 };
 export { type CPUUsage, CPUUsageIntervalLag };
 export * from "./replication.js";
+export type {
+	LogLike,
+	LogResultsIterator,
+	SharedLogLike,
+	SharedLogReplicationIndexLike,
+} from "./like.js";
 export {
 	type ReplicationRangeIndexable,
 	ReplicationRangeIndexableU32,
@@ -352,6 +362,7 @@ export type SharedLogOptions<
 	keep?: (
 		entry: ShallowOrFullEntry<T> | EntryReplicated<R>,
 	) => Promise<boolean> | boolean;
+	sync?: SyncOptions<R>;
 	syncronizer?: SynchronizerConstructor<R>;
 	timeUntilRoleMaturity?: number;
 	waitForReplicatorTimeout?: number;
@@ -2037,6 +2048,7 @@ export class SharedLog<
 				rangeIndex: this._replicationRangeIndex,
 				rpc: this.rpc,
 				coordinateToHash: this.coordinateToHash,
+				sync: options?.sync,
 			});
 		} else {
 			if (
@@ -2048,6 +2060,7 @@ export class SharedLog<
 					rpc: this.rpc,
 					entryIndex: this.entryCoordinatesIndex,
 					coordinateToHash: this.coordinateToHash,
+					sync: options?.sync,
 				});
 			} else {
 				if (this.domain.resolution === "u32") {
@@ -2063,6 +2076,7 @@ export class SharedLog<
 					rangeIndex: this._replicationRangeIndex,
 					rpc: this.rpc,
 					coordinateToHash: this.coordinateToHash,
+					sync: options?.sync,
 				}) as Syncronizer<R>;
 			}
 		}

package/src/like.ts

@@ -0,0 +1,84 @@
+import type { PublicSignKey } from "@peerbit/crypto";
+import type {
+	CountOptions,
+	IndexIterator,
+	IterateOptions,
+} from "@peerbit/indexer-interface";
+import type { Entry } from "@peerbit/log";
+import type { ShallowEntry } from "@peerbit/log";
+import type { ReplicationOptions, ReplicationRangeIndexable } from "./index.js";
+
+export type LogBlocksLike = {
+	has: (hash: string) => Promise<boolean> | boolean;
+};
+
+export type LogResultsIterator<T> = {
+	close: () => void | Promise<void>;
+	next: (amount: number) => T[] | Promise<T[]>;
+	done: () => boolean | undefined;
+	all: () => T[] | Promise<T[]>;
+};
+
+export type LogLike<T = any> = {
+	idString?: string;
+	length: number;
+	get: (
+		hash: string,
+		options?: any,
+	) => Promise<Entry<T> | undefined> | Entry<T> | undefined;
+	has: (hash: string) => Promise<boolean> | boolean;
+	getHeads: (resolve?: boolean) => LogResultsIterator<Entry<T> | ShallowEntry>;
+	toArray: () => Promise<Entry<T>[]>;
+	blocks?: LogBlocksLike;
+};
+
+export type SharedLogReplicationIndexLike<R extends "u32" | "u64" = any> = {
+	iterate: (
+		request?: IterateOptions,
+	) => IndexIterator<ReplicationRangeIndexable<R>, undefined>;
+	count: (options?: CountOptions) => Promise<number> | number;
+	getSize?: () => Promise<number> | number;
+};
+
+export type SharedLogLike<T = any, R extends "u32" | "u64" = any> = {
+	closed?: boolean;
+	events: EventTarget;
+	log: LogLike<T>;
+	replicationIndex: SharedLogReplicationIndexLike<R>;
+	node?: { identity: { publicKey: PublicSignKey } };
+	getReplicators: () => Promise<Set<string>>;
+	waitForReplicator: (
+		publicKey: PublicSignKey,
+		options?: {
+			eager?: boolean;
+			roleAge?: number;
+			timeout?: number;
+			signal?: AbortSignal;
+		},
+	) => Promise<void>;
+	waitForReplicators: (options?: {
+		timeout?: number;
+		roleAge?: number;
+		coverageThreshold?: number;
+		waitForNewPeers?: boolean;
+		signal?: AbortSignal;
+	}) => Promise<void>;
+	replicate: (
+		rangeOrEntry?: ReplicationOptions<R> | any,
+		options?: {
+			reset?: boolean;
+			checkDuplicates?: boolean;
+			rebalance?: boolean;
+			mergeSegments?: boolean;
+		},
+	) => Promise<void | ReplicationRangeIndexable<R>[]>;
+	unreplicate: (rangeOrEntry?: { id: Uint8Array }[]) => Promise<void>;
+	calculateCoverage: (options?: {
+		start?: number | bigint;
+		end?: number | bigint;
+		roleAge?: number;
+	}) => Promise<number>;
+	getMyReplicationSegments: () => Promise<ReplicationRangeIndexable<R>[]>;
+	getAllReplicationSegments: () => Promise<ReplicationRangeIndexable<R>[]>;
+	close: () => Promise<void | boolean>;
+};

package/src/sync/index.ts

@@ -8,6 +8,24 @@ import type { Numbers } from "../integers.js";
 import type { TransportMessage } from "../message.js";
 import type { EntryReplicated, ReplicationRangeIndexable } from "../ranges.js";
 
+export type SyncPriorityFn<R extends "u32" | "u64"> = (
+	entry: EntryReplicated<R>,
+) => number;
+
+export type SyncOptions<R extends "u32" | "u64"> = {
+	/**
+	 * Higher numbers are synced first.
+	 * The callback should be fast and side-effect free.
+	 */
+	priority?: SyncPriorityFn<R>;
+
+	/**
+	 * When using rateless IBLT sync, optionally pre-sync up to this many
+	 * high-priority entries using the simple synchronizer.
+	 */
+	maxSimpleEntries?: number;
+};
+
 export type SynchronizerComponents<R extends "u32" | "u64"> = {
 	rpc: RPC<TransportMessage, TransportMessage>;
 	rangeIndex: Index<ReplicationRangeIndexable<R>, any>;
@@ -15,6 +33,7 @@ export type SynchronizerComponents<R extends "u32" | "u64"> = {
 	log: Log<any>;
 	coordinateToHash: Cache<string>;
 	numbers: Numbers<R>;
+	sync?: SyncOptions<R>;
 };
 export type SynchronizerConstructor<R extends "u32" | "u64"> = new (
 	properties: SynchronizerComponents<R>,

package/src/sync/rateless-iblt.ts

@@ -4,18 +4,24 @@ import { type PublicSignKey, randomBytes, toBase64 } from "@peerbit/crypto";
 import { type Index } from "@peerbit/indexer-interface";
 import type { Entry, Log } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
-import { DecoderWrapper, EncoderWrapper } from "@peerbit/riblt";
+import {
+	DecoderWrapper,
+	EncoderWrapper,
+	ready as ribltReady,
+} from "@peerbit/riblt";
 import type { RPC, RequestContext } from "@peerbit/rpc";
 import { SilentDelivery } from "@peerbit/stream-interface";
 import { type EntryWithRefs } from "../exchange-heads.js";
-import { type Numbers } from "../integers.js";
 import { TransportMessage } from "../message.js";
 import {
 	type EntryReplicated,
-	type ReplicationRangeIndexable,
 	matchEntriesInRangeQuery,
 } from "../ranges.js";
-import type { SyncableKey, Syncronizer } from "./index.js";
+import type {
+	SyncableKey,
+	SynchronizerComponents,
+	Syncronizer,
+} from "./index.js";
 import { SimpleSyncronizer } from "./simple.js";
 
 export const logger = loggerFn("peerbit:shared-log:rateless");
@@ -142,6 +148,7 @@ const buildEncoderOrDecoderFromRange = async <
 	entryIndex: Index<EntryReplicated<D>>,
 	type: T,
 ): Promise<E | false> => {
+	await ribltReady;
 	const encoder =
 		type === "encoder" ? new EncoderWrapper() : new DecoderWrapper();
 
@@ -180,6 +187,13 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	simple: SimpleSyncronizer<D>;
 
 	startedOrCompletedSynchronizations: Cache<string>;
+	private localRangeEncoderCacheVersion = 0;
+	private localRangeEncoderCache: Map<
+		string,
+		{ encoder: EncoderWrapper; version: number; lastUsed: number }
+	> = new Map();
+	private localRangeEncoderCacheMax = 2;
+
 	ingoingSyncProcesses: Map<
 		string,
 		{
@@ -207,14 +221,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	>;
 
 	constructor(
-		readonly properties: {
-			rpc: RPC<TransportMessage, TransportMessage>;
-			rangeIndex: Index<ReplicationRangeIndexable<D>, any>;
-			entryIndex: Index<EntryReplicated<D>, any>;
-			log: Log<any>;
-			coordinateToHash: Cache<string>;
-			numbers: Numbers<D>;
-		},
+		readonly properties: SynchronizerComponents<D>,
 	) {
 		this.simple = new SimpleSyncronizer(properties);
 		this.outgoingSyncProcesses = new Map();
@@ -222,6 +229,91 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
 	}
 
+	private clearLocalRangeEncoderCache() {
+		for (const [, cached] of this.localRangeEncoderCache) {
+			cached.encoder.free();
+		}
+		this.localRangeEncoderCache.clear();
+	}
+
+	private invalidateLocalRangeEncoderCache() {
+		this.localRangeEncoderCacheVersion += 1;
+		this.clearLocalRangeEncoderCache();
+	}
+
+	private localRangeEncoderCacheKey(ranges: {
+		start1: NumberOrBigint;
+		end1: NumberOrBigint;
+		start2: NumberOrBigint;
+		end2: NumberOrBigint;
+	}) {
+		return `${String(ranges.start1)}:${String(ranges.end1)}:${String(
+			ranges.start2,
+		)}:${String(ranges.end2)}`;
+	}
+
+	private decoderFromCachedEncoder(encoder: EncoderWrapper): DecoderWrapper {
+		const clone = encoder.clone();
+		const decoder = clone.to_decoder();
+		clone.free();
+		return decoder;
+	}
+
+	private async getLocalDecoderForRange(ranges: {
+		start1: NumberOrBigint;
+		end1: NumberOrBigint;
+		start2: NumberOrBigint;
+		end2: NumberOrBigint;
+	}): Promise<DecoderWrapper | false> {
+		const key = this.localRangeEncoderCacheKey(ranges);
+		const cached = this.localRangeEncoderCache.get(key);
+		if (cached && cached.version === this.localRangeEncoderCacheVersion) {
+			cached.lastUsed = Date.now();
+			return this.decoderFromCachedEncoder(cached.encoder);
+		}
+
+		const encoder = (await buildEncoderOrDecoderFromRange(
+			ranges,
+			this.properties.entryIndex,
+			"encoder",
+		)) as EncoderWrapper | false;
+		if (!encoder) {
+			return false;
+		}
+
+		const now = Date.now();
+		const existing = this.localRangeEncoderCache.get(key);
+		if (existing) {
+			existing.encoder.free();
+		}
+		this.localRangeEncoderCache.set(key, {
+			encoder,
+			version: this.localRangeEncoderCacheVersion,
+			lastUsed: now,
+		});
+
+		while (this.localRangeEncoderCache.size > this.localRangeEncoderCacheMax) {
+			let oldestKey: string | undefined;
+			let oldestUsed = Number.POSITIVE_INFINITY;
+			for (const [candidateKey, value] of this.localRangeEncoderCache) {
+				if (value.lastUsed < oldestUsed) {
+					oldestUsed = value.lastUsed;
+					oldestKey = candidateKey;
+				}
+			}
+			if (!oldestKey) {
+				break;
+			}
+			const victim = this.localRangeEncoderCache.get(oldestKey);
+			if (victim) {
+				victim.encoder.free();
+			}
+			this.localRangeEncoderCache.delete(oldestKey);
+		}
+
+		return this.decoderFromCachedEncoder(encoder);
+	}
+
 	async onMaybeMissingEntries(properties: {
 		entries: Map<string, EntryReplicated<D>>;
 		targets: string[];
@@ -233,7 +325,6 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		//   such as those assigned to range boundaries.
 
 		let entriesToSyncNaively: Map<string, EntryReplicated<D>> = new Map();
-		let allCoordinatesToSyncWithIblt: bigint[] = [];
 		let minSyncIbltSize = 333; // TODO: make configurable
 		let maxSyncWithSimpleMethod = 1e3;
 
@@ -246,15 +337,57 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			return;
 		}
 
-		// Mixed strategy for larger batches
+		const nonBoundaryEntries: EntryReplicated<D>[] = [];
 		for (const entry of properties.entries.values()) {
 			if (entry.assignedToRangeBoundary) {
 				entriesToSyncNaively.set(entry.hash, entry);
 			} else {
-				allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
+				nonBoundaryEntries.push(entry);
 			}
 		}
 
+		const priorityFn = this.properties.sync?.priority;
+		const maxSimpleEntries = this.properties.sync?.maxSimpleEntries;
+		const maxAdditionalNaive =
+			priorityFn &&
+			typeof maxSimpleEntries === "number" &&
+			Number.isFinite(maxSimpleEntries) &&
+			maxSimpleEntries > 0
+				? Math.max(
+						0,
+						Math.min(
+							Math.floor(maxSimpleEntries),
+							maxSyncWithSimpleMethod - entriesToSyncNaively.size,
+						),
+					)
+				: 0;
+
+		if (priorityFn && maxAdditionalNaive > 0 && nonBoundaryEntries.length > 0) {
+			let index = 0;
+			const scored: {
+				entry: EntryReplicated<D>;
+				index: number;
+				priority: number;
+			}[] = [];
+			for (const entry of nonBoundaryEntries) {
+				const priorityValue = priorityFn(entry);
+				scored.push({
+					entry,
+					index,
+					priority: Number.isFinite(priorityValue) ? priorityValue : 0,
+				});
+				index += 1;
+			}
+			scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
+			for (const { entry } of scored.slice(0, maxAdditionalNaive)) {
+				entriesToSyncNaively.set(entry.hash, entry);
+			}
+		}
+
+		let allCoordinatesToSyncWithIblt = nonBoundaryEntries
+			.filter((entry) => !entriesToSyncNaively.has(entry.hash))
+			.map((entry) => coerceBigInt(entry.hashNumber));
+
 		if (entriesToSyncNaively.size > 0) {
 			// If there are special-case entries, sync them simply in parallel
 			await this.simple.onMaybeMissingEntries({
@@ -277,6 +410,8 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			return;
 		}
 
+		await ribltReady;
+
 		const sortedEntries = allCoordinatesToSyncWithIblt.sort((a, b) => {
 			if (a > b) {
 				return 1;
@@ -399,16 +534,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			this.startedOrCompletedSynchronizations.add(syncId);
 
 			const wrapped = message.end < message.start;
-			const decoder = await buildEncoderOrDecoderFromRange(
-				{
-					start1: message.start,
-					end1: wrapped ? this.properties.numbers.maxValue : message.end,
-					start2: 0n,
-					end2: wrapped ? message.end : 0n,
-				},
-				this.properties.entryIndex,
-				"decoder",
-			);
+			const decoder = await this.getLocalDecoderForRange({
+				start1: message.start,
+				end1: wrapped ? this.properties.numbers.maxValue : message.end,
+				start2: 0n,
+				end2: wrapped ? message.end : 0n,
+			});
 
 			if (!decoder) {
 				await this.simple.rpc.send(
@@ -613,10 +744,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	}
 
 	onEntryAdded(entry: Entry<any>): void {
+		this.invalidateLocalRangeEncoderCache();
 		return this.simple.onEntryAdded(entry);
 	}
 
 	onEntryRemoved(hash: string) {
+		this.invalidateLocalRangeEncoderCache();
 		return this.simple.onEntryRemoved(hash);
 	}
 
@@ -635,6 +768,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		for (const [, obj] of this.outgoingSyncProcesses) {
 			obj.free();
 		}
+		this.clearLocalRangeEncoderCache();
 		return this.simple.close();
 	}
 

package/src/sync/simple.ts

@@ -16,7 +16,7 @@ import {
 } from "../exchange-heads.js";
 import { TransportMessage } from "../message.js";
 import type { EntryReplicated } from "../ranges.js";
-import type { SyncableKey, Syncronizer } from "./index.js";
+import type { SyncableKey, SyncOptions, Syncronizer } from "./index.js";
 
 @variant([0, 1])
 export class RequestMaybeSync extends TransportMessage {
@@ -57,7 +57,7 @@ const getHashesFromSymbols = async (
 	coordinateToHash: Cache<string>,
 ) => {
 	let queries: IntegerCompare[] = [];
-	let batchSize = 1; // TODO arg
+	let batchSize = 128; // TODO arg
 	let results = new Set<string>();
 	const handleBatch = async (end = false) => {
 		if (queries.length >= batchSize || (end && queries.length > 0)) {
@@ -109,6 +109,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 	log: Log<any>;
 	entryIndex: Index<EntryReplicated<R>, any>;
 	coordinateToHash: Cache<string>;
+	private syncOptions?: SyncOptions<R>;
 
 	// Syncing and dedeplucation work
 	syncMoreInterval?: ReturnType<typeof setTimeout>;
@@ -120,6 +121,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 		entryIndex: Index<EntryReplicated<R>, any>;
 		log: Log<any>;
 		coordinateToHash: Cache<string>;
+		sync?: SyncOptions<R>;
 	}) {
 		this.syncInFlightQueue = new Map();
 		this.syncInFlightQueueInverted = new Map();
@@ -128,14 +130,35 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 		this.log = properties.log;
 		this.entryIndex = properties.entryIndex;
 		this.coordinateToHash = properties.coordinateToHash;
+		this.syncOptions = properties.sync;
 	}
 
 	onMaybeMissingEntries(properties: {
 		entries: Map<string, EntryReplicated<R>>;
 		targets: string[];
 	}): Promise<void> {
+		let hashes: string[];
+		const priorityFn = this.syncOptions?.priority;
+		if (priorityFn) {
+			let index = 0;
+			const scored: { hash: string; index: number; priority: number }[] = [];
+			for (const [hash, entry] of properties.entries) {
+				const priorityValue = priorityFn(entry);
+				scored.push({
+					hash,
+					index,
+					priority: Number.isFinite(priorityValue) ? priorityValue : 0,
+				});
+				index += 1;
+			}
+			scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
+			hashes = scored.map((x) => x.hash);
+		} else {
+			hashes = [...properties.entries.keys()];
+		}
+
 		return this.rpc.send(
-			new RequestMaybeSync({ hashes: [...properties.entries.keys()] }),
+			new RequestMaybeSync({ hashes }),
 			{
 				priority: 1,
 				mode: new SilentDelivery({ to: properties.targets, redundancy: 1 }),