@peerbit/shared-log 13.2.14 → 13.2.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/shared-log",
-  "version": "13.2.14",
+  "version": "13.2.16",
   "description": "Shared log",
   "sideEffects": false,
   "type": "module",
@@ -61,27 +61,27 @@
     "pidusage": "^4.0.1",
     "pino": "^9.4.0",
     "uint8arrays": "^5.1.0",
+    "@peerbit/any-store": "2.2.13",
     "@peerbit/blocks": "4.2.7",
-    "@peerbit/blocks-interface": "2.1.3",
     "@peerbit/crypto": "3.1.4",
-    "@peerbit/cache": "3.1.1",
-    "@peerbit/any-store": "2.2.13",
     "@peerbit/diagnostics": "0.0.1",
-    "@peerbit/indexer-sqlite3": "3.0.11",
+    "@peerbit/cache": "3.1.1",
     "@peerbit/indexer-interface": "3.0.8",
-    "@peerbit/logger": "2.0.1",
+    "@peerbit/blocks-interface": "2.1.3",
+    "@peerbit/indexer-sqlite3": "3.0.11",
     "@peerbit/log": "6.2.10",
-    "@peerbit/program": "6.0.40",
+    "@peerbit/logger": "2.0.1",
+    "@peerbit/rpc": "6.1.9",
     "@peerbit/pubsub-interface": "5.1.7",
     "@peerbit/riblt": "1.2.0",
-    "@peerbit/pubsub": "5.3.5",
-    "@peerbit/rpc": "6.1.8",
+    "@peerbit/program": "6.0.41",
+    "@peerbit/time": "3.0.1",
     "@peerbit/stream-interface": "6.0.13",
-    "@peerbit/time": "3.0.1"
+    "@peerbit/pubsub": "5.3.6"
   },
   "optionalDependencies": {
-    "@peerbit/shared-log-rust": "0.1.3",
-    "@peerbit/native-backbone": "0.2.3"
+    "@peerbit/native-backbone": "0.2.3",
+    "@peerbit/shared-log-rust": "0.1.3"
   },
   "devDependencies": {
     "@types/libsodium-wrappers": "^0.7.14",
@@ -89,8 +89,8 @@
     "uuid": "^11.1.1",
     "@peerbit/any-store-rust": "0.1.3",
     "@peerbit/indexer-rust": "1.0.6",
-    "@peerbit/test-utils": "3.1.12",
-    "peerbit": "5.3.12"
+    "@peerbit/test-utils": "3.1.13",
+    "peerbit": "5.3.13"
   },
   "repository": {
     "type": "git",

package/src/exchange-heads.ts

@@ -1160,26 +1160,32 @@ export class ResponseIPruneV2 extends TransportMessage {
 
 const MAX_EXCHANGE_MESSAGE_SIZE = 1e5; // 100kb. Too large size might not be faster (even if we can do 5mb)
 export const MAX_RAW_EXCHANGE_MESSAGE_SIZE = 512 * 1024;
+// Hash-resolved full entries and raw JS blocks remain reachable while a yielded
+// message is being sent, so bound their count-based lookahead. Caller-owned
+// Entry[] values and an individual entry's bytes are not byte-bounded here;
+// native metadata-only planning can retain its larger batch.
+const EXCHANGE_HEADS_PAYLOAD_RESOLVE_BATCH_SIZE = 16;
 export const EXCHANGE_HEADS_RESOLVE_BATCH_SIZE = 256;
 
-export const createExchangeHeadsMessages = async function* (
+const createExchangeHeadsMessagesWithVisited = async function* (
 	log: Log<any>,
 	heads: Entry<any>[] | string[] | Set<string>,
+	visitedHeads: Set<string>,
 ): AsyncGenerator<ExchangeHeadsMessage<any>, void, void> {
 	let size = 0;
 	let current: EntryWithRefs<any>[] = [];
-	const visitedHeads = new Set<string>();
 	const headArray = Array.isArray(heads) ? heads : [...heads];
-	const canUseNativeReferenceGids = headArray.length === 1;
+	const canUseNativeReferenceGids =
+		headArray.length === 1 && visitedHeads.size === 0;
 
 	for (
 		let offset = 0;
 		offset < headArray.length;
-		offset += EXCHANGE_HEADS_RESOLVE_BATCH_SIZE
+		offset += EXCHANGE_HEADS_PAYLOAD_RESOLVE_BATCH_SIZE
 	) {
 		const headBatch = headArray.slice(
 			offset,
-			offset + EXCHANGE_HEADS_RESOLVE_BATCH_SIZE,
+			offset + EXCHANGE_HEADS_PAYLOAD_RESOLVE_BATCH_SIZE,
 		);
 		const nativeReferenceRowsByPosition =
 			canUseNativeReferenceGids === false
@@ -1192,6 +1198,7 @@ export const createExchangeHeadsMessages = async function* (
 		);
 		for (let i = 0; i < resolvedHeads.length; i++) {
 			const entry = resolvedHeads[i];
+			resolvedHeads[i] = undefined;
 			if (!entry) {
 				continue; // missing this entry, could be deleted while iterating
 			}
@@ -1290,6 +1297,13 @@ export const createExchangeHeadsMessages = async function* (
 	}
 };
 
+export const createExchangeHeadsMessages = async function* (
+	log: Log<any>,
+	heads: Entry<any>[] | string[] | Set<string>,
+): AsyncGenerator<ExchangeHeadsMessage<any>, void, void> {
+	yield* createExchangeHeadsMessagesWithVisited(log, heads, new Set<string>());
+};
+
 export const createRawExchangeHeadsMessages = async function* (
 	log: Log<any>,
 	heads: string[] | Set<string>,
@@ -1313,30 +1327,44 @@ export const createRawExchangeHeadsMessages = async function* (
 			});
 		}
 	};
+	const flushCurrent = () => {
+		if (current.length === 0) {
+			return undefined;
+		}
+		emitJsBlockBytes(current, size);
+		const message = new RawExchangeHeadsMessage({ heads: current });
+		size = 0;
+		current = [];
+		return message;
+	};
 
-	for (let offset = 0; offset < headArray.length; offset += EXCHANGE_HEADS_RESOLVE_BATCH_SIZE) {
+	for (
+		let offset = 0;
+		offset < headArray.length;
+		offset += EXCHANGE_HEADS_PAYLOAD_RESOLVE_BATCH_SIZE
+	) {
 		const headBatch = headArray.slice(
 			offset,
-			offset + EXCHANGE_HEADS_RESOLVE_BATCH_SIZE,
+			offset + EXCHANGE_HEADS_PAYLOAD_RESOLVE_BATCH_SIZE,
 		);
 		const nativeReferenceRowsByPosition = getNativeReferenceRowsByHeadInput(
 			log,
 			headBatch,
 			visitedHeads,
 		);
-		if (!nativeReferenceRowsByPosition) {
-			for await (const message of createExchangeHeadsMessages(log, headBatch)) {
-				yield message;
+		const blockRows = nativeReferenceRowsByPosition
+			? await resolveExchangeHeadBlocks(log, headBatch, visitedHeads)
+			: undefined;
+		if (!nativeReferenceRowsByPosition || !blockRows) {
+			const pending = flushCurrent();
+			if (pending) {
+				yield pending;
 			}
-			continue;
-		}
-		const blockRows = await resolveExchangeHeadBlocks(
-			log,
-			headBatch,
-			visitedHeads,
-		);
-		if (!blockRows) {
-			for await (const message of createExchangeHeadsMessages(log, headBatch)) {
+			for await (const message of createExchangeHeadsMessagesWithVisited(
+				log,
+				headBatch,
+				visitedHeads,
+			)) {
 				yield message;
 			}
 			continue;
@@ -1344,6 +1372,7 @@ export const createRawExchangeHeadsMessages = async function* (
 
 		for (let i = 0; i < blockRows.length; i++) {
 			const block = blockRows[i];
+			blockRows[i] = undefined;
 			if (!block) {
 				continue;
 			}
@@ -1376,16 +1405,16 @@ export const createRawExchangeHeadsMessages = async function* (
 			);
 			size += block.bytes.byteLength;
 			if (size > MAX_RAW_EXCHANGE_MESSAGE_SIZE) {
-				emitJsBlockBytes(current, size);
-				size = 0;
-				yield new RawExchangeHeadsMessage({ heads: current });
-				current = [];
+				const pending = flushCurrent();
+				if (pending) {
+					yield pending;
+				}
 			}
 		}
 	}
-	if (current.length > 0) {
-		emitJsBlockBytes(current, size);
-		yield new RawExchangeHeadsMessage({ heads: current });
+	const pending = flushCurrent();
+	if (pending) {
+		yield pending;
 	}
 };
 

package/src/index.ts

@@ -16342,6 +16342,8 @@ export class SharedLog<
 				sync: options?.sync,
 				isEntryRecentlyKnownByPeer: (hash, peer, maxAgeMs) =>
 					this.isEntryRecentlyKnownByPeer(hash, peer, maxAgeMs),
+				peerSupportsRawExchangeHeads: (peer) =>
+					this.peerSupportsRawExchangeHeads(peer),
 				sendRawExchangeHeads,
 			});
 		} else {
@@ -16359,6 +16361,8 @@ export class SharedLog<
 					sync: options?.sync,
 					isEntryRecentlyKnownByPeer: (hash, peer, maxAgeMs) =>
 						this.isEntryRecentlyKnownByPeer(hash, peer, maxAgeMs),
+					peerSupportsRawExchangeHeads: (peer) =>
+						this.peerSupportsRawExchangeHeads(peer),
 					sendRawExchangeHeads,
 				});
 			} else {
@@ -16381,6 +16385,8 @@ export class SharedLog<
 					sync: options?.sync,
 					isEntryRecentlyKnownByPeer: (hash, peer, maxAgeMs) =>
 						this.isEntryRecentlyKnownByPeer(hash, peer, maxAgeMs),
+					peerSupportsRawExchangeHeads: (peer) =>
+						this.peerSupportsRawExchangeHeads(peer),
 					sendRawExchangeHeads,
 				}) as Syncronizer<R>;
 			}

package/src/sync/index.ts

@@ -172,6 +172,7 @@ export type SynchronizerComponents<R extends "u32" | "u64"> = {
 		peer: string,
 		maxAgeMs: number,
 	) => boolean;
+	peerSupportsRawExchangeHeads?: (peer: string) => boolean;
 	sendRawExchangeHeads?: RawExchangeHeadsSender;
 };
 export type SynchronizerConstructor<R extends "u32" | "u64"> = new (

package/src/sync/simple.ts

@@ -558,6 +558,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 		peer: string,
 		maxAgeMs: number,
 	) => boolean;
+	private peerSupportsRawExchangeHeads?: (peer: string) => boolean;
 	private sendRawExchangeHeads?: RawExchangeHeadsSender;
 	private recentlySentExchangeHeads: Map<string, Map<string, number>>;
 	private pendingMaybeSyncResponses: Map<
@@ -602,6 +603,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 			peer: string,
 			maxAgeMs: number,
 		) => boolean;
+		peerSupportsRawExchangeHeads?: (peer: string) => boolean;
 		sendRawExchangeHeads?: RawExchangeHeadsSender;
 	}) {
 		this.syncInFlightQueue = new Map();
@@ -638,6 +640,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 		this.resolveHashListForSymbols = properties.resolveHashListForSymbols;
 		this.syncOptions = properties.sync;
 		this.isEntryRecentlyKnownByPeer = properties.isEntryRecentlyKnownByPeer;
+		this.peerSupportsRawExchangeHeads = properties.peerSupportsRawExchangeHeads;
 		this.sendRawExchangeHeads = properties.sendRawExchangeHeads;
 		this.recentlySentExchangeHeads = new Map();
 		this.pendingMaybeSyncResponses = new Map();
@@ -3958,13 +3961,16 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 					this.maxCoordinatesPerMessage,
 				);
 				for (const target of targets) {
+					const requestRawExchangeHeads =
+						this.syncOptions?.rawExchangeHeads === true &&
+						this.peerSupportsRawExchangeHeads?.(target) === true;
 					for (const chunk of chunks) {
 						if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
 							break;
 						}
 						try {
 							await this.rpc.send(
-								this.syncOptions?.rawExchangeHeads
+								requestRawExchangeHeads
 									? new RequestMaybeSyncCoordinateCapabilities({
 											hashNumbers: chunk,
 										})
@@ -3993,13 +3999,16 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
 			if (stringHashes.length > 0) {
 				const chunks = this.chunk(stringHashes, this.maxHashesPerMessage);
 				for (const target of targets) {
+					const requestRawExchangeHeads =
+						this.syncOptions?.rawExchangeHeads === true &&
+						this.peerSupportsRawExchangeHeads?.(target) === true;
 					for (const chunk of chunks) {
 						if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
 							break;
 						}
 						try {
 							await this.rpc.send(
-								this.syncOptions?.rawExchangeHeads
+								requestRawExchangeHeads
 									? new ResponseMaybeSyncCapabilities({
 											hashes: chunk,
 										})