@peerbit/shared-log 12.3.4 → 12.3.5-3f16953

package/src/pid.ts

@@ -63,16 +63,34 @@ export class PIDReplicationController {
 			errorCoverageUnmodified;
 
 		const errorFromEven = 1 / peerCount - currentFactor;
+		// When the network is under-covered (`totalFactor < 1`) balancing "down" (negative
+		// error) can further reduce coverage and force constrained peers (memory/CPU limited)
+		// to take boundary assignments that exceed their budgets.
+		//
+		// Use a soft clamp: only suppress negative balance strongly when the coverage deficit
+		// is material. This avoids oscillations around `totalFactor ~= 1`.
+		const coverageDeficit = Math.max(0, errorCoverageUnmodified); // ~= max(0, 1 - totalFactor)
+		const negativeBalanceScale =
+			coverageDeficit <= 0
+				? 1
+				: 1 - Math.min(1, coverageDeficit / 0.1); // full clamp at 10% deficit
+		const errorFromEvenForBalance =
+			errorFromEven >= 0 ? errorFromEven : errorFromEven * negativeBalanceScale;
 
 		const balanceErrorScaler = this.maxMemoryLimit
 			? Math.abs(errorMemory)
 			: 1 - Math.abs(errorCoverage);
 
-		const errorBalance = (this.maxMemoryLimit ? errorMemory > 0 : true)
-			? errorFromEven > 0
-				? balanceErrorScaler * errorFromEven
+		// Balance should be symmetric (allow negative error) so a peer can *reduce*
+		// participation when peerCount increases. Otherwise early joiners can get
+		// "stuck" over-replicating even after new peers join (no memory/CPU limits).
+		const errorBalance = this.maxMemoryLimit
+			? // Only balance when we have spare memory headroom. When memory is
+				// constrained (`errorMemory < 0`) the memory term will dominate anyway.
+				errorMemory > 0
+				? balanceErrorScaler * errorFromEvenForBalance
 				: 0
-			: 0;
+			: balanceErrorScaler * errorFromEvenForBalance;
 
 		const errorCPU = peerCount > 1 ? -1 * (cpuUsage || 0) : 0;
 

package/src/ranges.ts

@@ -2565,13 +2565,17 @@ export const debounceAggregationChanges = <
 			let aggregated: Map<string, ReplicationChange<T>> = new Map();
 			return {
 				add: (change: ReplicationChange<T>) => {
-					const prev = aggregated.get(change.range.idString);
+					// Keep different change types for the same segment id. In particular, range
+					// updates produce a `replaced` + `added` pair; collapsing by id would drop the
+					// "removed" portion and prevent correct rebalancing/pruning.
+					const key = `${change.type}:${change.range.idString}`;
+					const prev = aggregated.get(key);
 					if (prev) {
 						if (prev.range.timestamp < change.range.timestamp) {
-							aggregated.set(change.range.idString, change);
+							aggregated.set(key, change);
 						}
 					} else {
-						aggregated.set(change.range.idString, change);
+						aggregated.set(key, change);
 					}
 				},
 				delete: (key: string) => {

package/src/sync/rateless-iblt.ts

@@ -1,7 +1,13 @@
 import { field, variant, vec } from "@dao-xyz/borsh";
 import { Cache } from "@peerbit/cache";
 import { type PublicSignKey, randomBytes, toBase64 } from "@peerbit/crypto";
-import { type Index } from "@peerbit/indexer-interface";
+import {
+	And,
+	type Index,
+	IntegerCompare,
+	Or,
+	type Query,
+} from "@peerbit/indexer-interface";
 import type { Entry, Log } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import {
@@ -13,7 +19,7 @@ import type { RPC, RequestContext } from "@peerbit/rpc";
 import { SilentDelivery } from "@peerbit/stream-interface";
 import { type EntryWithRefs } from "../exchange-heads.js";
 import { TransportMessage } from "../message.js";
-import { type EntryReplicated, matchEntriesInRangeQuery } from "../ranges.js";
+import { type EntryReplicated } from "../ranges.js";
 import type {
 	SyncableKey,
 	SynchronizerComponents,
@@ -131,6 +137,50 @@ export interface SSymbol {
 	symbol: bigint;
 }
 
+const matchEntriesByHashNumberInRangeQuery = (range: {
+	start1: number | bigint;
+	end1: number | bigint;
+	start2: number | bigint;
+	end2: number | bigint;
+}): Query => {
+	const c1 = new And([
+		new IntegerCompare({
+			key: "hashNumber",
+			compare: "gte",
+			value: range.start1,
+		}),
+		new IntegerCompare({
+			key: "hashNumber",
+			compare: "lt",
+			value: range.end1,
+		}),
+	]);
+
+	// if range2 has length 0 or range 2 is equal to range 1 only make one query
+	if (
+		range.start2 === range.end2 ||
+		(range.start1 === range.start2 && range.end1 === range.end2)
+	) {
+		return c1;
+	}
+
+	return new Or([
+		c1,
+		new And([
+			new IntegerCompare({
+				key: "hashNumber",
+				compare: "gte",
+				value: range.start2,
+			}),
+			new IntegerCompare({
+				key: "hashNumber",
+				compare: "lt",
+				value: range.end2,
+			}),
+		]),
+	]);
+};
+
 const buildEncoderOrDecoderFromRange = async <
 	T extends "encoder" | "decoder",
 	E = T extends "encoder" ? EncoderWrapper : DecoderWrapper,
@@ -152,7 +202,8 @@ const buildEncoderOrDecoderFromRange = async <
 	const entries = await entryIndex
 		.iterate(
 			{
-				query: matchEntriesInRangeQuery({
+				// Range sync for IBLT is done in hashNumber space.
+				query: matchEntriesByHashNumberInRangeQuery({
 					end1: ranges.end1,
 					start1: ranges.start1,
 					end2: ranges.end2,
@@ -478,9 +529,13 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
 			}
 		}
 
+		// For smaller sets, the original `sqrt(n)` heuristic can occasionally under-provision
+		// low-degree symbols early, causing an unnecessary `MoreSymbols` round-trip. Use a
+		// small floor to make small-delta syncs more reliable without affecting large-n behavior.
 		let initialSymbols = Math.round(
 			Math.sqrt(allCoordinatesToSyncWithIblt.length),
 		); // TODO choose better
+		initialSymbols = Math.max(64, initialSymbols);
 		for (let i = 0; i < initialSymbols; i++) {
 			startSync.symbols.push(
 				new SymbolSerialized(encoder.produce_next_coded_symbol()),