@peerbit/shared-log 12.1.3 → 12.2.0-874976b

package/src/sync/index.ts

@@ -8,6 +8,24 @@ import type { Numbers } from "../integers.js";
 import type { TransportMessage } from "../message.js";
 import type { EntryReplicated, ReplicationRangeIndexable } from "../ranges.js";
 
+export type SyncPriorityFn<R extends "u32" | "u64"> = (
+	entry: EntryReplicated<R>,
+) => number;
+
+export type SyncOptions<R extends "u32" | "u64"> = {
+	/**
+	 * Higher numbers are synced first.
+	 * The callback should be fast and side-effect free.
+	 */
+	priority?: SyncPriorityFn<R>;
+
+	/**
+	 * When using rateless IBLT sync, optionally pre-sync up to this many
+	 * high-priority entries using the simple synchronizer.
+	 */
+	maxSimpleEntries?: number;
+};
+
 export type SynchronizerComponents<R extends "u32" | "u64"> = {
 	rpc: RPC<TransportMessage, TransportMessage>;
 	rangeIndex: Index<ReplicationRangeIndexable<R>, any>;
@@ -15,6 +33,7 @@ export type SynchronizerComponents<R extends "u32" | "u64"> = {
 	log: Log<any>;
 	coordinateToHash: Cache<string>;
 	numbers: Numbers<R>;
+	sync?: SyncOptions<R>;
 };
 export type SynchronizerConstructor<R extends "u32" | "u64"> = new (
 	properties: SynchronizerComponents<R>,

package/src/sync/rateless-iblt.ts

@@ -4,18 +4,24 @@ import { type PublicSignKey, randomBytes, toBase64 } from "@peerbit/crypto";
 import { type Index } from "@peerbit/indexer-interface";
 import type { Entry, Log } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
-import { DecoderWrapper, EncoderWrapper } from "@peerbit/riblt";
+import {
+	DecoderWrapper,
+	EncoderWrapper,
+	ready as ribltReady,
+} from "@peerbit/riblt";
 import type { RPC, RequestContext } from "@peerbit/rpc";
 import { SilentDelivery } from "@peerbit/stream-interface";
 import { type EntryWithRefs } from "../exchange-heads.js";
-import { type Numbers } from "../integers.js";
 import { TransportMessage } from "../message.js";
 import {
 	type EntryReplicated,
-	type ReplicationRangeIndexable,
 	matchEntriesInRangeQuery,
 } from "../ranges.js";
-import type { SyncableKey, Syncronizer } from "./index.js";
+import type {
+	SyncableKey,
+	SynchronizerComponents,
+	Syncronizer,
+} from "./index.js";
 import { SimpleSyncronizer } from "./simple.js";
 
 export const logger = loggerFn("peerbit:shared-log:rateless");
@@ -142,6 +148,7 @@ const buildEncoderOrDecoderFromRange = async <
 	entryIndex: Index<EntryReplicated<D>>,
 	type: T,
 ): Promise<E | false> => {
+	await ribltReady;
 	const encoder =
 		type === "encoder" ? new EncoderWrapper() : new DecoderWrapper();
 
@@ -207,14 +214,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	>;
 
 	constructor(
-		readonly properties: {
-			rpc: RPC<TransportMessage, TransportMessage>;
-			rangeIndex: Index<ReplicationRangeIndexable<D>, any>;
-			entryIndex: Index<EntryReplicated<D>, any>;
-			log: Log<any>;
-			coordinateToHash: Cache<string>;
-			numbers: Numbers<D>;
-		},
+		readonly properties: SynchronizerComponents<D>,
 	) {
 		this.simple = new SimpleSyncronizer(properties);
 		this.outgoingSyncProcesses = new Map();
@@ -233,7 +233,6 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		//   such as those assigned to range boundaries.
 
 		let entriesToSyncNaively: Map<string, EntryReplicated<D>> = new Map();
-		let allCoordinatesToSyncWithIblt: bigint[] = [];
 		let minSyncIbltSize = 333; // TODO: make configurable
 		let maxSyncWithSimpleMethod = 1e3;
 
@@ -246,15 +245,57 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			return;
 		}
 
-		// Mixed strategy for larger batches
+		const nonBoundaryEntries: EntryReplicated<D>[] = [];
 		for (const entry of properties.entries.values()) {
 			if (entry.assignedToRangeBoundary) {
 				entriesToSyncNaively.set(entry.hash, entry);
 			} else {
-				allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
+				nonBoundaryEntries.push(entry);
 			}
 		}
 
+		const priorityFn = this.properties.sync?.priority;
+		const maxSimpleEntries = this.properties.sync?.maxSimpleEntries;
+		const maxAdditionalNaive =
+			priorityFn &&
+			typeof maxSimpleEntries === "number" &&
+			Number.isFinite(maxSimpleEntries) &&
+			maxSimpleEntries > 0
+				? Math.max(
+						0,
+						Math.min(
+							Math.floor(maxSimpleEntries),
+							maxSyncWithSimpleMethod - entriesToSyncNaively.size,
+						),
+					)
+				: 0;
+
+		if (priorityFn && maxAdditionalNaive > 0 && nonBoundaryEntries.length > 0) {
+			let index = 0;
+			const scored: {
+				entry: EntryReplicated<D>;
+				index: number;
+				priority: number;
+			}[] = [];
+			for (const entry of nonBoundaryEntries) {
+				const priorityValue = priorityFn(entry);
+				scored.push({
+					entry,
+					index,
+					priority: Number.isFinite(priorityValue) ? priorityValue : 0,
+				});
+				index += 1;
+			}
+			scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
+			for (const { entry } of scored.slice(0, maxAdditionalNaive)) {
+				entriesToSyncNaively.set(entry.hash, entry);
+			}
+		}
+
+		let allCoordinatesToSyncWithIblt = nonBoundaryEntries
+			.filter((entry) => !entriesToSyncNaively.has(entry.hash))
+			.map((entry) => coerceBigInt(entry.hashNumber));
+
 		if (entriesToSyncNaively.size > 0) {
 			// If there are special-case entries, sync them simply in parallel
 			await this.simple.onMaybeMissingEntries({
@@ -277,6 +318,8 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			return;
 		}
 
+		await ribltReady;
+
 		const sortedEntries = allCoordinatesToSyncWithIblt.sort((a, b) => {
 			if (a > b) {
 				return 1;

package/src/sync/simple.ts

@@ -16,7 +16,7 @@ import {
 } from "../exchange-heads.js";
 import { TransportMessage } from "../message.js";
 import type { EntryReplicated } from "../ranges.js";
-import type { SyncableKey, Syncronizer } from "./index.js";
+import type { SyncableKey, SyncOptions, Syncronizer } from "./index.js";
 
 @variant([0, 1])
 export class RequestMaybeSync extends TransportMessage {
@@ -57,7 +57,7 @@ const getHashesFromSymbols = async (
 	coordinateToHash: Cache<string>,
 ) => {
 	let queries: IntegerCompare[] = [];
-	let batchSize = 1; // TODO arg
+	let batchSize = 128; // TODO arg
 	let results = new Set<string>();
 	const handleBatch = async (end = false) => {
 		if (queries.length >= batchSize || (end && queries.length > 0)) {
@@ -109,6 +109,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 	log: Log<any>;
 	entryIndex: Index<EntryReplicated<R>, any>;
 	coordinateToHash: Cache<string>;
+	private syncOptions?: SyncOptions<R>;
 
 	// Syncing and dedeplucation work
 	syncMoreInterval?: ReturnType<typeof setTimeout>;
@@ -120,6 +121,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 		entryIndex: Index<EntryReplicated<R>, any>;
 		log: Log<any>;
 		coordinateToHash: Cache<string>;
+		sync?: SyncOptions<R>;
 	}) {
 		this.syncInFlightQueue = new Map();
 		this.syncInFlightQueueInverted = new Map();
@@ -128,14 +130,35 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 		this.log = properties.log;
 		this.entryIndex = properties.entryIndex;
 		this.coordinateToHash = properties.coordinateToHash;
+		this.syncOptions = properties.sync;
 	}
 
 	onMaybeMissingEntries(properties: {
 		entries: Map<string, EntryReplicated<R>>;
 		targets: string[];
 	}): Promise<void> {
+		let hashes: string[];
+		const priorityFn = this.syncOptions?.priority;
+		if (priorityFn) {
+			let index = 0;
+			const scored: { hash: string; index: number; priority: number }[] = [];
+			for (const [hash, entry] of properties.entries) {
+				const priorityValue = priorityFn(entry);
+				scored.push({
+					hash,
+					index,
+					priority: Number.isFinite(priorityValue) ? priorityValue : 0,
+				});
+				index += 1;
+			}
+			scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
+			hashes = scored.map((x) => x.hash);
+		} else {
+			hashes = [...properties.entries.keys()];
+		}
+
 		return this.rpc.send(
-			new RequestMaybeSync({ hashes: [...properties.entries.keys()] }),
+			new RequestMaybeSync({ hashes }),
 			{
 				priority: 1,
 				mode: new SilentDelivery({ to: properties.targets, redundancy: 1 }),