@peerbit/shared-log 12.3.5 → 13.0.0

package/src/pid.ts

@@ -63,16 +63,34 @@ export class PIDReplicationController {
 			errorCoverageUnmodified;
 
 		const errorFromEven = 1 / peerCount - currentFactor;
+		// When the network is under-covered (`totalFactor < 1`) balancing "down" (negative
+		// error) can further reduce coverage and force constrained peers (memory/CPU limited)
+		// to take boundary assignments that exceed their budgets.
+		//
+		// Use a soft clamp: only suppress negative balance strongly when the coverage deficit
+		// is material. This avoids oscillations around `totalFactor ~= 1`.
+		const coverageDeficit = Math.max(0, errorCoverageUnmodified); // ~= max(0, 1 - totalFactor)
+		const negativeBalanceScale =
+			coverageDeficit <= 0
+				? 1
+				: 1 - Math.min(1, coverageDeficit / 0.1); // full clamp at 10% deficit
+		const errorFromEvenForBalance =
+			errorFromEven >= 0 ? errorFromEven : errorFromEven * negativeBalanceScale;
 
 		const balanceErrorScaler = this.maxMemoryLimit
 			? Math.abs(errorMemory)
 			: 1 - Math.abs(errorCoverage);
 
-		const errorBalance = (this.maxMemoryLimit ? errorMemory > 0 : true)
-			? errorFromEven > 0
-				? balanceErrorScaler * errorFromEven
+		// Balance should be symmetric (allow negative error) so a peer can *reduce*
+		// participation when peerCount increases. Otherwise early joiners can get
+		// "stuck" over-replicating even after new peers join (no memory/CPU limits).
+		const errorBalance = this.maxMemoryLimit
+			? // Only balance when we have spare memory headroom. When memory is
+				// constrained (`errorMemory < 0`) the memory term will dominate anyway.
+				errorMemory > 0
+				? balanceErrorScaler * errorFromEvenForBalance
 				: 0
-			: 0;
+			: balanceErrorScaler * errorFromEvenForBalance;
 
 		const errorCPU = peerCount > 1 ? -1 * (cpuUsage || 0) : 0;
 

package/src/ranges.ts

@@ -2565,13 +2565,17 @@ export const debounceAggregationChanges = <
 			let aggregated: Map<string, ReplicationChange<T>> = new Map();
 			return {
 				add: (change: ReplicationChange<T>) => {
-					const prev = aggregated.get(change.range.idString);
+					// Keep different change types for the same segment id. In particular, range
+					// updates produce a `replaced` + `added` pair; collapsing by id would drop the
+					// "removed" portion and prevent correct rebalancing/pruning.
+					const key = `${change.type}:${change.range.idString}`;
+					const prev = aggregated.get(key);
 					if (prev) {
 						if (prev.range.timestamp < change.range.timestamp) {
-							aggregated.set(change.range.idString, change);
+							aggregated.set(key, change);
 						}
 					} else {
-						aggregated.set(change.range.idString, change);
+						aggregated.set(key, change);
 					}
 				},
 				delete: (key: string) => {
@@ -2703,8 +2707,14 @@ export const toRebalance = <R extends "u32" | "u64">(
 		| ReplicationChanges<ReplicationRangeIndexable<R>>[],
 	index: Index<EntryReplicated<R>>,
 	rebalanceHistory: Cache<string>,
+	options?: { forceFresh?: boolean },
 ): AsyncIterable<EntryReplicated<R>> => {
-	const change = mergeReplicationChanges(changeOrChanges, rebalanceHistory);
+	const change = options?.forceFresh
+		? (Array.isArray(changeOrChanges[0])
+				? (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>[])
+						.flat()
+				: (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>))
+		: mergeReplicationChanges(changeOrChanges, rebalanceHistory);
 	return {
 		[Symbol.asyncIterator]: async function* () {
 			const iterator = index.iterate({

package/src/sync/index.ts

@@ -24,6 +24,30 @@ export type SyncOptions<R extends "u32" | "u64"> = {
 	 * high-priority entries using the simple synchronizer.
 	 */
 	maxSimpleEntries?: number;
+
+	/**
+	 * Maximum number of hash strings in one simple sync message.
+	 */
+	maxSimpleHashesPerMessage?: number;
+
+	/**
+	 * Maximum number of coordinates in one simple sync coordinate message.
+	 */
+	maxSimpleCoordinatesPerMessage?: number;
+
+	/**
+	 * Maximum number of hashes tracked per convergent repair session target.
+	 * Large sessions still dispatch all entries, but only this many are tracked
+	 * for deterministic completion metadata.
+	 */
+	maxConvergentTrackedHashes?: number;
+
+	/**
+	 * Maximum number of candidate entries buffered per target before the
+	 * background repair sweep dispatches a maybe-sync batch.
+	 * Larger values reduce orchestration overhead but increase per-target memory.
+	 */
+	repairSweepTargetBufferSize?: number;
 };
 
 export type SynchronizerComponents<R extends "u32" | "u64"> = {
@@ -41,7 +65,35 @@ export type SynchronizerConstructor<R extends "u32" | "u64"> = new (
 
 export type SyncableKey = string | bigint; // hash or coordinate
 
+export type RepairSessionMode = "best-effort" | "convergent";
+
+export type RepairSessionResult = {
+	target: string;
+	requested: number;
+	resolved: number;
+	unresolved: string[];
+	attempts: number;
+	durationMs: number;
+	completed: boolean;
+	requestedTotal?: number;
+	truncated?: boolean;
+};
+
+export type RepairSession = {
+	id: string;
+	done: Promise<RepairSessionResult[]>;
+	cancel: () => void;
+};
+
 export interface Syncronizer<R extends "u32" | "u64"> {
+	startRepairSession(properties: {
+		entries: Map<string, EntryReplicated<R>>;
+		targets: string[];
+		mode?: RepairSessionMode;
+		timeoutMs?: number;
+		retryIntervalsMs?: number[];
+	}): RepairSession;
+
 	onMaybeMissingEntries(properties: {
 		entries: Map<string, EntryReplicated<R>>;
 		targets: string[];
@@ -59,7 +111,7 @@ export interface Syncronizer<R extends "u32" | "u64"> {
 
 	onEntryAdded(entry: Entry<any>): void;
 	onEntryRemoved(hash: string): void;
-	onPeerDisconnected(key: PublicSignKey): void;
+	onPeerDisconnected(key: PublicSignKey | string): void;
 
 	open(): Promise<void> | void;
 	close(): Promise<void> | void;

package/src/sync/rateless-iblt.ts

@@ -1,7 +1,13 @@
 import { field, variant, vec } from "@dao-xyz/borsh";
 import { Cache } from "@peerbit/cache";
 import { type PublicSignKey, randomBytes, toBase64 } from "@peerbit/crypto";
-import { type Index } from "@peerbit/indexer-interface";
+import {
+	And,
+	type Index,
+	IntegerCompare,
+	Or,
+	type Query,
+} from "@peerbit/indexer-interface";
 import type { Entry, Log } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import {
@@ -13,8 +19,11 @@ import type { RPC, RequestContext } from "@peerbit/rpc";
 import { SilentDelivery } from "@peerbit/stream-interface";
 import { type EntryWithRefs } from "../exchange-heads.js";
 import { TransportMessage } from "../message.js";
-import { type EntryReplicated, matchEntriesInRangeQuery } from "../ranges.js";
+import { type EntryReplicated } from "../ranges.js";
 import type {
+	RepairSession,
+	RepairSessionMode,
+	RepairSessionResult,
 	SyncableKey,
 	SynchronizerComponents,
 	Syncronizer,
@@ -49,6 +58,10 @@ const getSyncIdString = (message: { syncId: Uint8Array }) => {
 	return toBase64(message.syncId);
 };
 
+const DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS = 30_000;
+const DEFAULT_CONVERGENT_RETRY_INTERVALS_MS = [0, 1_000, 3_000, 7_000];
+const DEFAULT_MAX_CONVERGENT_TRACKED_HASHES = 4_096;
+
 @variant([3, 0])
 export class StartSync extends TransportMessage {
 	@field({ type: Uint8Array })
@@ -131,6 +144,50 @@ export interface SSymbol {
 	symbol: bigint;
 }
 
+const matchEntriesByHashNumberInRangeQuery = (range: {
+	start1: number | bigint;
+	end1: number | bigint;
+	start2: number | bigint;
+	end2: number | bigint;
+}): Query => {
+	const c1 = new And([
+		new IntegerCompare({
+			key: "hashNumber",
+			compare: "gte",
+			value: range.start1,
+		}),
+		new IntegerCompare({
+			key: "hashNumber",
+			compare: "lt",
+			value: range.end1,
+		}),
+	]);
+
+	// if range2 has length 0 or range 2 is equal to range 1 only make one query
+	if (
+		range.start2 === range.end2 ||
+		(range.start1 === range.start2 && range.end1 === range.end2)
+	) {
+		return c1;
+	}
+
+	return new Or([
+		c1,
+		new And([
+			new IntegerCompare({
+				key: "hashNumber",
+				compare: "gte",
+				value: range.start2,
+			}),
+			new IntegerCompare({
+				key: "hashNumber",
+				compare: "lt",
+				value: range.end2,
+			}),
+		]),
+	]);
+};
+
 const buildEncoderOrDecoderFromRange = async <
 	T extends "encoder" | "decoder",
 	E = T extends "encoder" ? EncoderWrapper : DecoderWrapper,
@@ -152,7 +209,8 @@ const buildEncoderOrDecoderFromRange = async <
 	const entries = await entryIndex
 		.iterate(
 			{
-				query: matchEntriesInRangeQuery({
+				// Range sync for IBLT is done in hashNumber space.
+				query: matchEntriesByHashNumberInRangeQuery({
 					end1: ranges.end1,
 					start1: ranges.start1,
 					end2: ranges.end2,
@@ -182,6 +240,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 	implements Syncronizer<D>
 {
 	simple: SimpleSyncronizer<D>;
+	private repairSessionCounter: number;
 
 	startedOrCompletedSynchronizations: Cache<string>;
 	private localRangeEncoderCacheVersion = 0;
@@ -219,11 +278,179 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 
 	constructor(readonly properties: SynchronizerComponents<D>) {
 		this.simple = new SimpleSyncronizer(properties);
+		this.repairSessionCounter = 0;
 		this.outgoingSyncProcesses = new Map();
 		this.ingoingSyncProcesses = new Map();
 		this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
 	}
 
+	private get maxConvergentTrackedHashes() {
+		const value = this.properties.sync?.maxConvergentTrackedHashes;
+		return value && Number.isFinite(value) && value > 0
+			? Math.floor(value)
+			: DEFAULT_MAX_CONVERGENT_TRACKED_HASHES;
+	}
+
+	private normalizeRetryIntervals(retryIntervalsMs?: number[]): number[] {
+		if (!retryIntervalsMs || retryIntervalsMs.length === 0) {
+			return [...DEFAULT_CONVERGENT_RETRY_INTERVALS_MS];
+		}
+
+		return [...retryIntervalsMs]
+			.map((x) => Math.max(0, Math.floor(x)))
+			.filter((x, i, arr) => arr.indexOf(x) === i)
+			.sort((a, b) => a - b);
+	}
+
+	private getPrioritizedEntries(entries: Map<string, EntryReplicated<D>>) {
+		const priorityFn = this.properties.sync?.priority;
+		if (!priorityFn) {
+			return [...entries.values()];
+		}
+
+		let index = 0;
+		const scored: { entry: EntryReplicated<D>; index: number; priority: number }[] =
+			[];
+		for (const entry of entries.values()) {
+			const priorityValue = priorityFn(entry);
+			scored.push({
+				entry,
+				index,
+				priority: Number.isFinite(priorityValue) ? priorityValue : 0,
+			});
+			index += 1;
+		}
+		scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
+		return scored.map((x) => x.entry);
+	}
+
+	startRepairSession(properties: {
+		entries: Map<string, EntryReplicated<D>>;
+		targets: string[];
+		mode?: RepairSessionMode;
+		timeoutMs?: number;
+		retryIntervalsMs?: number[];
+	}): RepairSession {
+		const mode = properties.mode ?? "best-effort";
+		const targets = [...new Set(properties.targets)];
+		const timeoutMs = Math.max(
+			1,
+			Math.floor(properties.timeoutMs ?? DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS),
+		);
+		const retryIntervalsMs = this.normalizeRetryIntervals(properties.retryIntervalsMs);
+		const trackedLimit = this.maxConvergentTrackedHashes;
+		const requestedHashes = [...properties.entries.keys()];
+		const requestedHashesTracked = requestedHashes.slice(0, trackedLimit);
+		const truncated = requestedHashesTracked.length < requestedHashes.length;
+
+		if (mode === "convergent") {
+			if (properties.entries.size <= trackedLimit) {
+				return this.simple.startRepairSession({
+					...properties,
+					mode: "convergent",
+					timeoutMs,
+					retryIntervalsMs,
+				});
+			}
+
+			const id = `rateless-repair-${++this.repairSessionCounter}`;
+			const startedAt = Date.now();
+			const prioritized = this.getPrioritizedEntries(properties.entries);
+			const trackedEntries = new Map<string, EntryReplicated<D>>();
+			for (const entry of prioritized.slice(0, trackedLimit)) {
+				trackedEntries.set(entry.hash, entry);
+			}
+
+			let cancelled = false;
+			const trackedSession = this.simple.startRepairSession({
+				entries: trackedEntries,
+				targets,
+				mode: "convergent",
+				timeoutMs,
+				retryIntervalsMs,
+			});
+
+			const runDispatchSchedule = async () => {
+				let previousDelay = 0;
+				for (const delayMs of retryIntervalsMs) {
+					if (cancelled) {
+						return;
+					}
+					const elapsed = Date.now() - startedAt;
+					if (elapsed >= timeoutMs) {
+						return;
+					}
+					const waitMs = Math.max(0, delayMs - previousDelay);
+					previousDelay = delayMs;
+					if (waitMs > 0) {
+						await new Promise<void>((resolve) => {
+							const timer = setTimeout(resolve, waitMs);
+							timer.unref?.();
+						});
+					}
+					if (cancelled) {
+						return;
+					}
+					try {
+						await this.onMaybeMissingEntries({
+							entries: properties.entries,
+							targets,
+						});
+					} catch {
+						// Best-effort schedule: tracked session timeout/result decides completion.
+					}
+				}
+			};
+
+			const done = (async (): Promise<RepairSessionResult[]> => {
+				await runDispatchSchedule();
+				const trackedResults = await trackedSession.done;
+				return trackedResults.map((result) => ({
+					...result,
+					requestedTotal: requestedHashes.length,
+					truncated: true,
+				}));
+			})();
+
+			return {
+				id,
+				done,
+				cancel: () => {
+					cancelled = true;
+					trackedSession.cancel();
+				},
+			};
+		}
+
+		const id = `rateless-repair-${++this.repairSessionCounter}`;
+		const startedAt = Date.now();
+		const done = (async (): Promise<RepairSessionResult[]> => {
+			await this.onMaybeMissingEntries({
+				entries: properties.entries,
+				targets,
+			});
+			const durationMs = Date.now() - startedAt;
+			return targets.map((target) => ({
+				target,
+				requested: requestedHashesTracked.length,
+				resolved: 0,
+				unresolved: [...requestedHashesTracked],
+				attempts: 1,
+				durationMs,
+				completed: false,
+				requestedTotal: requestedHashes.length,
+				truncated,
+			}));
+		})();
+		return {
+			id,
+			done,
+			cancel: () => {
+				// no-op: best-effort dispatch does not maintain cancelable session state
+			},
+		};
+	}
+
 	private clearLocalRangeEncoderCache() {
 		for (const [, cached] of this.localRangeEncoderCache) {
 			cached.encoder.free();
@@ -313,6 +540,8 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		entries: Map<string, EntryReplicated<D>>;
 		targets: string[];
 	}): Promise<void> {
+		// NOTE: this method is best-effort dispatch, not a per-hash convergence API.
+		// It may require follow-up repair rounds under churn/loss to fully close all gaps.
 		// Strategy:
 		// - For small sets, prefer the simple synchronizer to reduce complexity and avoid
 		//   IBLT overhead on tiny batches.
@@ -478,9 +707,13 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			}
 		}
 
+		// For smaller sets, the original `sqrt(n)` heuristic can occasionally under-provision
+		// low-degree symbols early, causing an unnecessary `MoreSymbols` round-trip. Use a
+		// small floor to make small-delta syncs more reliable without affecting large-n behavior.
 		let initialSymbols = Math.round(
 			Math.sqrt(allCoordinatesToSyncWithIblt.length),
 		); // TODO choose better
+		initialSymbols = Math.max(64, initialSymbols);
 		for (let i = 0; i < initialSymbols; i++) {
 			startSync.symbols.push(
 				new SymbolSerialized(encoder.produce_next_coded_symbol()),
@@ -770,7 +1003,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 		return this.simple.onEntryRemoved(hash);
 	}
 
-	onPeerDisconnected(key: PublicSignKey) {
+	onPeerDisconnected(key: PublicSignKey | string) {
 		return this.simple.onPeerDisconnected(key);
 	}