@peerbit/shared-log 13.1.18 → 13.2.0

package/src/pid.ts

@@ -56,22 +56,32 @@ export class PIDReplicationController {
 			// prunes from repeatedly settling just above the hard budget.
 			const effectiveMemoryLimit =
 				this.maxMemoryLimit > 0 ? this.maxMemoryLimit * 0.95 : 0;
-			errorMemory =
-				currentFactor > 0 && memoryUsage > 0
-					? Math.max(
-							Math.min(1, effectiveMemoryLimit / estimatedTotalSize),
-							0,
-						) - currentFactor
-					: 0;
+			if (effectiveMemoryLimit <= 0) {
+				// A zero storage budget can never hold data; drive the factor to zero
+				// instead of leaving the memory term neutral.
+				errorMemory = -currentFactor;
+			} else if (currentFactor > 0 && memoryUsage > 0) {
+				errorMemory =
+					Math.max(Math.min(1, effectiveMemoryLimit / estimatedTotalSize), 0) -
+					currentFactor;
+			} else {
+				errorMemory = 0;
+			}
 			// Math.max(Math.min((this.maxMemoryLimit - memoryUsage) / 100e5, 1), -1)// Math.min(Math.max((this.maxMemoryLimit - memoryUsage, 0) / 10e5, 0), 1);
 		}
 
 		const errorCoverageUnmodified = Math.min(1 - totalFactor, 1);
 		let errorCoverage =
-			(this.maxMemoryLimit ? 1 - Math.sqrt(Math.abs(errorMemory)) : 1) *
+			(this.maxMemoryLimit != null
+				? this.maxMemoryLimit > 0
+					? 1 - Math.sqrt(Math.abs(errorMemory))
+					: 0
+				: 1) *
 			errorCoverageUnmodified;
 
 		const errorFromEven = 1 / peerCount - currentFactor;
+		const hasMemoryHeadroom =
+			this.maxMemoryLimit != null && this.maxMemoryLimit > 0 && errorMemory > 0;
 		// When the network is under-covered (`totalFactor < 1`) balancing "down" (negative
 		// error) can further reduce coverage and force constrained peers (memory/CPU limited)
 		// to take boundary assignments that exceed their budgets.
@@ -81,11 +91,12 @@ export class PIDReplicationController {
 		const coverageDeficit = Math.max(0, errorCoverageUnmodified); // ~= max(0, 1 - totalFactor)
 		const negativeBalanceScale =
 			coverageDeficit <= 0 ? 1 : 1 - Math.min(1, coverageDeficit / 0.1); // full clamp at 10% deficit
-		const errorFromEvenForBalance =
+		let errorFromEvenForBalance =
 			errorFromEven >= 0 ? errorFromEven : errorFromEven * negativeBalanceScale;
+		if (hasMemoryHeadroom && coverageDeficit > 0 && errorFromEvenForBalance < 0) {
+			errorFromEvenForBalance = 0;
+		}
 
-		const hasMemoryHeadroom =
-			this.maxMemoryLimit != null && this.maxMemoryLimit > 0 && errorMemory > 0;
 		if (hasMemoryHeadroom && errorFromEvenForBalance > 0) {
 			// Coverage surplus often means another peer has not pruned yet. Do not let
 			// that transient surplus cancel a constrained peer that is still below an
@@ -93,7 +104,7 @@ export class PIDReplicationController {
 			errorCoverage = Math.max(errorCoverage, 0);
 		}
 
-		const balanceErrorScaler = this.maxMemoryLimit
+		const balanceErrorScaler = this.maxMemoryLimit != null
 			? hasMemoryHeadroom
 				? Math.max(
 						Math.abs(errorMemory),
@@ -105,7 +116,7 @@ export class PIDReplicationController {
 		// Balance should be symmetric (allow negative error) so a peer can *reduce*
 		// participation when peerCount increases. Otherwise early joiners can get
 		// "stuck" over-replicating even after new peers join (no memory/CPU limits).
-		const errorBalance = this.maxMemoryLimit
+		const errorBalance = this.maxMemoryLimit != null
 			? // Only balance when we have spare memory headroom. When memory is
 				// constrained (`errorMemory < 0`) the memory term will dominate anyway.
 				errorMemory > 0

package/src/ranges.ts

@@ -96,6 +96,7 @@ export interface EntryReplicated<R extends "u32" | "u64"> {
 	wallTime: bigint;
 	assignedToRangeBoundary: boolean;
 	get meta(): ShallowMeta;
+	getMetaBytes(): Uint8Array;
 }
 
 export const isEntryReplicated = (x: any): x is EntryReplicated<any> => {
@@ -130,22 +131,33 @@ export class EntryReplicatedU32 implements EntryReplicated<"u32"> {
 	constructor(properties: {
 		coordinates: number[];
 		hash: string;
-		meta: Meta;
+		meta?: Meta | ShallowMeta;
+		metaBytes?: Uint8Array;
+		gid?: string;
+		wallTime?: bigint;
 		assignedToRangeBoundary: boolean;
 		hashNumber: number;
 	}) {
+		if (!properties.meta && !properties.metaBytes) {
+			throw new Error("Expected meta or metaBytes");
+		}
+		if (!properties.meta && (properties.gid == null || properties.wallTime == null)) {
+			throw new Error("Expected gid and wallTime with metaBytes");
+		}
 		this.coordinates = properties.coordinates;
 		this.hash = properties.hash;
-		this.gid = properties.meta.gid;
-		this.wallTime = properties.meta.clock.timestamp.wallTime;
+		this.gid = properties.meta?.gid ?? properties.gid!;
+		this.wallTime =
+			properties.meta?.clock.timestamp.wallTime ?? properties.wallTime!;
 		this.hashNumber = properties.hashNumber;
-		const shallow =
-			properties.meta instanceof Meta
-				? new ShallowMeta(properties.meta)
-				: properties.meta;
-		this._meta = serialize(shallow);
-		this._metaResolved = deserialize(this._meta, ShallowMeta);
-		this._metaResolved = properties.meta;
+		this._meta =
+			properties.metaBytes ??
+			serialize(
+				properties.meta instanceof Meta
+					? new ShallowMeta(properties.meta)
+					: properties.meta!,
+			);
+		this._metaResolved = properties.meta as ShallowMeta;
 		this.assignedToRangeBoundary = properties.assignedToRangeBoundary;
 	}
 
@@ -155,6 +167,10 @@ export class EntryReplicatedU32 implements EntryReplicated<"u32"> {
 		}
 		return this._metaResolved;
 	}
+
+	getMetaBytes(): Uint8Array {
+		return this._meta;
+	}
 }
 
 @variant("entry-u64")
@@ -185,22 +201,33 @@ export class EntryReplicatedU64 implements EntryReplicated<"u64"> {
 	constructor(properties: {
 		coordinates: bigint[];
 		hash: string;
-		meta: Meta;
+		meta?: Meta | ShallowMeta;
+		metaBytes?: Uint8Array;
+		gid?: string;
+		wallTime?: bigint;
 		assignedToRangeBoundary: boolean;
 		hashNumber: bigint;
 	}) {
+		if (!properties.meta && !properties.metaBytes) {
+			throw new Error("Expected meta or metaBytes");
+		}
+		if (!properties.meta && (properties.gid == null || properties.wallTime == null)) {
+			throw new Error("Expected gid and wallTime with metaBytes");
+		}
 		this.coordinates = properties.coordinates;
 		this.hash = properties.hash;
 		this.hashNumber = properties.hashNumber;
-		this.gid = properties.meta.gid;
-		this.wallTime = properties.meta.clock.timestamp.wallTime;
-		const shallow =
-			properties.meta instanceof Meta
-				? new ShallowMeta(properties.meta)
-				: properties.meta;
-		this._meta = serialize(shallow);
-		this._metaResolved = deserialize(this._meta, ShallowMeta);
-		this._metaResolved = properties.meta;
+		this.gid = properties.meta?.gid ?? properties.gid!;
+		this.wallTime =
+			properties.meta?.clock.timestamp.wallTime ?? properties.wallTime!;
+		this._meta =
+			properties.metaBytes ??
+			serialize(
+				properties.meta instanceof Meta
+					? new ShallowMeta(properties.meta)
+					: properties.meta!,
+			);
+		this._metaResolved = properties.meta as ShallowMeta;
 		this.assignedToRangeBoundary = properties.assignedToRangeBoundary;
 	}
 
@@ -210,6 +237,10 @@ export class EntryReplicatedU64 implements EntryReplicated<"u64"> {
 		}
 		return this._metaResolved;
 	}
+
+	getMetaBytes(): Uint8Array {
+		return this._meta;
+	}
 }
 
 export const isReplicationRangeMessage = (
@@ -1310,6 +1341,11 @@ const allRangesContainingPoint = async <
 			},
 			options,
 		);
+		try {
+			(await firstIterator.all()).forEach((x) => allResults.push(x));
+		} finally {
+			await firstIterator.close();
+		}
 
 		const secondIterator = rects.iterate(
 			{
@@ -1318,10 +1354,11 @@ const allRangesContainingPoint = async <
 			},
 			options,
 		);
-
-		[...(await firstIterator.all()), ...(await secondIterator.all())].forEach(
-			(x) => allResults.push(x),
-		);
+		try {
+			(await secondIterator.all()).forEach((x) => allResults.push(x));
+		} finally {
+			await secondIterator.close();
+		}
 	}
 	return allResults;
 	/* return [...await iterateRangesContainingPoint(rects, points, options).all()]; */
@@ -2040,15 +2077,19 @@ const collectClosestAround = async <R extends "u32" | "u64">(
 	);
 
 	let visited = new Set<string>();
-	while (aroundIterator.done() !== true && done() !== true) {
-		const res = await aroundIterator.next(100);
-		for (const rect of res) {
-			visited.add(rect.value.idString);
-			collector(rect.value, isMatured(rect.value, now, roleAge));
-			if (done()) {
-				return;
+	try {
+		while (aroundIterator.done() !== true && done() !== true) {
+			const res = await aroundIterator.next(100);
+			for (const rect of res) {
+				visited.add(rect.value.idString);
+				collector(rect.value, isMatured(rect.value, now, roleAge));
+				if (done()) {
+					return;
+				}
 			}
 		}
+	} finally {
+		await aroundIterator.close();
 	}
 };
 
@@ -2709,6 +2750,8 @@ export const mergeReplicationChanges = <R extends NumericType>(
 	return all;
 };
 
+const REBALANCE_ITERATOR_BATCH_SIZE = 1048;
+
 export const toRebalance = <R extends "u32" | "u64">(
 	changeOrChanges:
 		| ReplicationChanges<ReplicationRangeIndexable<R>>
@@ -2718,22 +2761,29 @@ export const toRebalance = <R extends "u32" | "u64">(
 	options?: { forceFresh?: boolean },
 ): AsyncIterable<EntryReplicated<R>> => {
 	const change = options?.forceFresh
-		? (Array.isArray(changeOrChanges[0])
-				? (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>[])
-						.flat()
-				: (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>))
+		? Array.isArray(changeOrChanges[0])
+			? (
+					changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>[]
+				).flat()
+			: (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>)
 		: mergeReplicationChanges(changeOrChanges, rebalanceHistory);
 	return {
 		[Symbol.asyncIterator]: async function* () {
 			const iterator = index.iterate({
 				query: createAssignedRangesQuery(change),
 			});
-
-			while (iterator.done() !== true) {
-				const entries = await iterator.all(); // TODO choose right batch sizes here for optimal memory usage / speed
-				for (const entry of entries) {
-					yield entry.value;
+			try {
+				while (iterator.done() !== true) {
+					const entries = await iterator.next(REBALANCE_ITERATOR_BATCH_SIZE);
+					if (entries.length === 0) {
+						break;
+					}
+					for (const entry of entries) {
+						yield entry.value;
+					}
 				}
+			} finally {
+				await iterator.close();
 			}
 		},
 	};

package/src/sync/index.ts

@@ -15,6 +15,11 @@ export type SyncPriorityFn<R extends "u32" | "u64"> = (
 	entry: EntryReplicated<R>,
 ) => number;
 
+export type SyncEntryCoordinates<R extends "u32" | "u64"> = Pick<
+	EntryReplicated<R>,
+	"assignedToRangeBoundary" | "hash" | "hashNumber"
+>;
+
 export type SyncOptions<R extends "u32" | "u64"> = {
 	/**
 	 * Orders entries inside a sync batch; higher numbers are selected first.
@@ -39,6 +44,22 @@ export type SyncOptions<R extends "u32" | "u64"> = {
 	 */
 	maxSimpleCoordinatesPerMessage?: number;
 
+	/**
+	 * Experimental sync path for peers known to support raw exchange-head
+	 * responses. When enabled, simple sync requests advertise raw-head support
+	 * and capable responders can avoid full Entry materialization before sending.
+	 */
+	rawExchangeHeads?: boolean;
+
+	/**
+	 * Experimental receive-side raw-head parsing mode for native backbone. When
+	 * enabled, signature verification happens while raw entries are decoded
+	 * instead of during the later native commit step. When unset, shared-log may
+	 * enable this automatically for retained native receive paths where planning
+	 * already indicates that the local peer is replicating.
+	 */
+	rawExchangeHeadsVerifySignaturesDuringPrepare?: boolean;
+
 	/**
 	 * Maximum number of hashes tracked per convergent repair session target.
 	 * Large sessions still dispatch all entries, but only this many are tracked
@@ -53,6 +74,15 @@ export type SyncOptions<R extends "u32" | "u64"> = {
 	 */
 	repairSweepTargetBufferSize?: number;
 
+	/**
+	 * Native wire receive fusion (requires nativeBackbone). Shared-log
+	 * registers its RPC topic with this session so raw exchange-head payloads
+	 * recognized by the native wire decoder stay in wasm memory: the TS borsh
+	 * decode of the entries and the JS-to-wasm copy of their block bytes are
+	 * skipped. Messages without a stash entry fall back to the regular path.
+	 */
+	nativeWireSync?: SharedLogNativeWireSync;
+
 	/**
 	 * Optional profiling callback. It is only invoked when provided, and should
 	 * avoid blocking because it runs inside sync hot paths.
@@ -60,6 +90,72 @@ export type SyncOptions<R extends "u32" | "u64"> = {
 	profile?: SyncProfileFn;
 };
 
+/**
+ * The per-node native wire-sync session surface shared-log consumes
+ * (implemented by `NativeBackboneWireSyncSession` in `@peerbit/native-backbone`;
+ * the same object also implements the `nativeWire` option of
+ * `@peerbit/stream`'s DirectStream).
+ */
+export type SharedLogNativeWireSync = {
+	/** Raw wasm session handle consumed by `prepareStashedRawReceive*`. */
+	handle: unknown;
+	registerTopic(topic: string): void;
+	unregisterTopic(topic: string): boolean;
+	stashedMeta(id: Uint8Array):
+		| {
+				hashes: string[];
+				gidRefrences: string[][];
+				byteLengths: Uint32Array;
+				reserved: Uint8Array;
+				payloadLength: number;
+		  }
+		| undefined;
+	stashedBlocks(
+		id: Uint8Array,
+		indexes?: Uint32Array,
+	): Uint8Array[] | undefined;
+	release(id: Uint8Array): boolean;
+};
+
+/**
+ * Fused raw exchange-heads sender provided by the shared log when the native
+ * backbone can serialize the sync payload in wasm (block bytes never
+ * materialize in JS). Resolves to the number of messages sent, or `undefined`
+ * when the fused path is unavailable for these hashes — the caller then falls
+ * back to the TS message path.
+ */
+export type RawExchangeHeadsSender = (
+	hashes: string[],
+	to: string[],
+	options?: { priority?: number },
+) => Promise<number | undefined>;
+
+export type HashSymbolInput = readonly bigint[] | BigUint64Array;
+
+export type HashSymbolResolver = (
+	symbols: HashSymbolInput,
+) =>
+	| ReadonlyMap<bigint, Iterable<string>>
+	| undefined
+	| Promise<ReadonlyMap<bigint, Iterable<string>> | undefined>;
+
+export type HashSymbolHashListResolver = (
+	symbols: HashSymbolInput,
+) =>
+	| Iterable<string>
+	| undefined
+	| Promise<Iterable<string> | undefined>;
+
+export type HashSymbolRangeResolver = (range: {
+	start1: bigint | number;
+	end1: bigint | number;
+	start2: bigint | number;
+	end2: bigint | number;
+}) =>
+	| Iterable<bigint | number>
+	| undefined
+	| Promise<Iterable<bigint | number> | undefined>;
+
 export type SynchronizerComponents<R extends "u32" | "u64"> = {
 	rpc: RPC<TransportMessage, TransportMessage>;
 	rangeIndex: Index<ReplicationRangeIndexable<R>, any>;
@@ -67,7 +163,16 @@ export type SynchronizerComponents<R extends "u32" | "u64"> = {
 	log: Log<any>;
 	coordinateToHash: Cache<string>;
 	numbers: Numbers<R>;
+	resolveHashesForSymbols?: HashSymbolResolver;
+	resolveHashListForSymbols?: HashSymbolHashListResolver;
+	resolveHashNumbersInRange?: HashSymbolRangeResolver;
 	sync?: SyncOptions<R>;
+	isEntryRecentlyKnownByPeer?: (
+		hash: string,
+		peer: string,
+		maxAgeMs: number,
+	) => boolean;
+	sendRawExchangeHeads?: RawExchangeHeadsSender;
 };
 export type SynchronizerConstructor<R extends "u32" | "u64"> = new (
 	properties: SynchronizerComponents<R>,
@@ -97,7 +202,7 @@ export type RepairSession = {
 
 export interface Syncronizer<R extends "u32" | "u64"> {
 	startRepairSession(properties: {
-		entries: Map<string, EntryReplicated<R>>;
+		entries: Map<string, SyncEntryCoordinates<R>>;
 		targets: string[];
 		mode?: RepairSessionMode;
 		timeoutMs?: number;
@@ -105,7 +210,7 @@ export interface Syncronizer<R extends "u32" | "u64"> {
 	}): RepairSession;
 
 	onMaybeMissingEntries(properties: {
-		entries: Map<string, EntryReplicated<R>>;
+		entries: Map<string, SyncEntryCoordinates<R>>;
 		targets: string[];
 	}): Promise<void> | void;
 
@@ -119,7 +224,15 @@ export interface Syncronizer<R extends "u32" | "u64"> {
 		from: PublicSignKey;
 	}): Promise<void> | void;
 
+	onReceivedEntryHashes?(properties: {
+		hashes: string[];
+		from: PublicSignKey;
+	}): Promise<void> | void;
+
+	onEntryAddedHashes?(hashes: string[]): void;
+	onEntryAddedHash?(hash: string): void;
 	onEntryAdded(entry: Entry<any>): void;
+	onEntryRemovedHashes?(hashes: string[]): void;
 	onEntryRemoved(hash: string): void;
 	onPeerDisconnected(key: PublicSignKey | string): void;