@peerbit/blocks 4.1.9 → 4.2.1

package/src/any-blockstore.ts

@@ -2,6 +2,7 @@ import { createStore } from "@peerbit/any-store";
 import { type AnyStore } from "@peerbit/any-store-interface";
 import {
 	type Blocks,
+	type GetOptions,
 	calculateRawCid,
 	cidifyString,
 	codecCodes,
@@ -16,6 +17,27 @@ import { waitFor } from "@peerbit/time";
 import { type Block, decode } from "multiformats/block";
 import * as raw from "multiformats/codecs/raw";
 
+const isPromiseLike = <T>(value: Promise<T> | T): value is Promise<T> =>
+	typeof (value as { then?: unknown })?.then === "function";
+
+type ImmutableBlockStore = AnyStore & {
+	putImmutable?: (key: string, value: Uint8Array) => Promise<void> | void;
+	putManyImmutable?: (
+		entries: Iterable<readonly [string, Uint8Array]>,
+	) => Promise<void> | void;
+};
+
+type NativeLogBlockStoreCarrier = {
+	getNativeLogBlockStoreHandle?: () => unknown;
+	getBlockResponsePayload?: (cid: string) => Uint8Array | undefined;
+};
+
+type BlockGetOptions = GetOptions & {
+	raw?: boolean;
+	links?: string[];
+	hasher?: any;
+};
+
 export class AnyBlockStore implements Blocks {
 	private _store: AnyStore;
 	private _opening: Promise<any>;
@@ -25,42 +47,84 @@ export class AnyBlockStore implements Blocks {
 		this._store = store;
 	}
 
+	getNativeLogBlockStoreHandle(): unknown {
+		return (this._store as NativeLogBlockStoreCarrier)
+			.getNativeLogBlockStoreHandle?.();
+	}
+
+	getBlockResponsePayload(cid: string): Uint8Array | undefined {
+		return (this._store as NativeLogBlockStoreCarrier).getBlockResponsePayload?.(
+			cid,
+		);
+	}
+
+	private async decodeStoredBytes(
+		cid: string,
+		bytes: Uint8Array,
+		options?: BlockGetOptions,
+	): Promise<Uint8Array> {
+		const cidObject = cidifyString(cid);
+		if (
+			cidObject.code === raw.code &&
+			(options?.hasher == null || options.hasher === defaultHasher)
+		) {
+			return bytes;
+		}
+		const codec = codecCodes[cidObject.code as keyof typeof codecCodes];
+		const block = await decode({
+			bytes,
+			codec,
+			hasher: options?.hasher || defaultHasher,
+		});
+		return (block as Block<Uint8Array, any, any, any>).bytes;
+	}
+
 	async get(
 		cid: string,
-		options?: {
-			raw?: boolean;
-			links?: string[];
-			hasher?: any;
-			remote: {
-				timeout?: number;
-			};
-		},
+		options?: BlockGetOptions,
 	): Promise<Uint8Array | undefined> {
-		const cidObject = cidifyString(cid);
 		try {
 			const bytes = await this._store.get(cid);
 			if (!bytes) {
 				return undefined;
 			}
+			return this.decodeStoredBytes(cid, bytes, options);
+		} catch (error: any) {
 			if (
-				cidObject.code === raw.code &&
-				(options?.hasher == null || options.hasher === defaultHasher)
+				typeof error?.code === "string" &&
+				error?.code?.indexOf("LEVEL_NOT_FOUND") !== -1
 			) {
-				return bytes;
+				return undefined;
+			}
+			throw error;
+		}
+	}
+
+	async getMany(
+		cids: string[],
+		options?: BlockGetOptions,
+	): Promise<Array<Uint8Array | undefined>> {
+		const store = this._store as AnyStore & {
+			getMany?: (keys: string[]) => Promise<Array<Uint8Array | undefined>>;
+		};
+		try {
+			if (typeof store.getMany === "function") {
+				const values = await store.getMany(cids);
+				return Promise.all(
+					values.map((bytes, index) =>
+						bytes
+							? this.decodeStoredBytes(cids[index]!, bytes, options)
+							: undefined,
+					),
+				);
 			}
-			const codec = (codecCodes as any)[cidObject.code];
-			const block = await decode({
-				bytes,
-				codec,
-				hasher: options?.hasher || defaultHasher,
-			});
-			return (block as Block<Uint8Array, any, any, any>).bytes;
+			return Promise.all(cids.map((cid) => this.get(cid, options)));
 		} catch (error: any) {
 			if (
 				typeof error?.code === "string" &&
 				error?.code?.indexOf("LEVEL_NOT_FOUND") !== -1
 			) {
-				return undefined;
+				return Promise.all(cids.map((cid) => this.get(cid, options)));
 			}
 			throw error;
 		}
@@ -75,13 +139,7 @@ export class AnyBlockStore implements Blocks {
 		try {
 			await this._store.put(put.cid, bbytes);
 		} catch (error: any) {
-			const status = await this._store.status();
-			if (
-				typeof error?.code === "string" &&
-				error.code === "LEVEL_DATABASE_NOT_OPEN" &&
-				this._closeController?.signal.aborted === true &&
-				(status === "closing" || status === "closed")
-			) {
+			if (await this.isClosingStorePutError(error)) {
 				// Late replication writes can outlive shutdown. At this point the
 				// backing store is intentionally closing, so report the deterministic
 				// CID while discarding the write instead of leaking an unhandled
@@ -93,10 +151,140 @@ export class AnyBlockStore implements Blocks {
 		return put.cid;
 	}
 
+	async putMany(
+		blocks: Array<Uint8Array | { block: Block<any, any, any, any>; cid: string }>,
+	): Promise<string[]> {
+		const puts = await Promise.all(
+			blocks.map((bytes) =>
+				bytes instanceof Uint8Array ? calculateRawCid(bytes) : bytes,
+			),
+		);
+		const store = this._store as AnyStore & {
+			putMany?: (
+				entries: Iterable<readonly [string, Uint8Array]>,
+			) => Promise<void> | void;
+		};
+		try {
+			if (puts.length === 1) {
+				const put = puts[0]!;
+				const result = this.putKnown(put.cid, put.block.bytes);
+				if (isPromiseLike(result)) {
+					await result;
+				}
+			} else if (typeof store.putMany === "function") {
+				await store.putMany(puts.map((put) => [put.cid, put.block.bytes] as const));
+			} else {
+				for (const put of puts) {
+					await this._store.put(put.cid, put.block.bytes);
+				}
+			}
+		} catch (error: any) {
+			if (await this.isClosingStorePutError(error)) {
+				return puts.map((put) => put.cid);
+			}
+			throw error;
+		}
+		return puts.map((put) => put.cid);
+	}
+
+	putKnown(cid: string, bytes: Uint8Array): Promise<string> | string {
+		try {
+			const immutableStore = this._store as ImmutableBlockStore;
+			const result = immutableStore.putImmutable
+				? immutableStore.putImmutable(cid, bytes)
+				: this._store.put(cid, bytes);
+			if (isPromiseLike(result)) {
+				return result
+					.then(() => cid)
+					.catch((error) => this.handleStorePutError(error, cid));
+			}
+		} catch (error: any) {
+			return this.handleStorePutError(error, cid);
+		}
+		return cid;
+	}
+
+	putKnownMany(
+		blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
+	): Promise<string[]> | string[] {
+		const store = this._store as AnyStore & {
+			putMany?: (
+				entries: Iterable<readonly [string, Uint8Array]>,
+			) => Promise<void> | void;
+			putManyImmutable?: (
+				entries: Iterable<readonly [string, Uint8Array]>,
+			) => Promise<void> | void;
+		};
+		if (blocks.length === 1) {
+			const [cid, bytes] = blocks[0]!;
+			const result = this.putKnown(cid, bytes);
+			return isPromiseLike(result) ? result.then(() => [cid]) : [cid];
+		}
+		return this.putKnownManyWithStoreBatch(blocks, store);
+	}
+
+	private async putKnownManyWithStoreBatch(
+		blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
+		store: AnyStore & {
+			putMany?: (
+				entries: Iterable<readonly [string, Uint8Array]>,
+			) => Promise<void> | void;
+			putManyImmutable?: (
+				entries: Iterable<readonly [string, Uint8Array]>,
+			) => Promise<void> | void;
+		},
+	): Promise<string[]> {
+		try {
+			if (typeof store.putManyImmutable === "function") {
+				await store.putManyImmutable(blocks);
+			} else if (typeof store.putMany === "function") {
+				await store.putMany(blocks);
+			} else {
+				for (const [cid, bytes] of blocks) {
+					await this._store.put(cid, bytes);
+				}
+			}
+		} catch (error: any) {
+			if (await this.isClosingStorePutError(error)) {
+				return blocks.map(([cid]) => cid);
+			}
+			throw error;
+		}
+		return blocks.map(([cid]) => cid);
+	}
+
+	private async handleStorePutError<T>(error: any, value: T): Promise<T> {
+		if (await this.isClosingStorePutError(error)) {
+			return value;
+		}
+		throw error;
+	}
+
+	private async isClosingStorePutError(error: any): Promise<boolean> {
+		const status = await this._store.status();
+		return (
+			typeof error?.code === "string" &&
+			error.code === "LEVEL_DATABASE_NOT_OPEN" &&
+			this._closeController?.signal.aborted === true &&
+			(status === "closing" || status === "closed")
+		);
+	}
+
 	async rm(cid: string): Promise<void> {
 		await this._store.del(cid);
 	}
 
+	async rmMany(cids: string[]): Promise<number> {
+		const store = this._store as AnyStore & {
+			delMany?: (keys: string[]) => Promise<number>;
+		};
+		if (typeof store.delMany === "function") {
+			return store.delMany(cids);
+		}
+		await Promise.all(cids.map((cid) => this._store.del(cid)));
+		return cids.length;
+	}
+
 	async *iterator(): AsyncGenerator<[string, Uint8Array], void, void> {
 		for await (const [key, value] of this._store.iterator()) {
 			yield [key, value];
@@ -104,7 +292,35 @@ export class AnyBlockStore implements Blocks {
 	}
 
 	async has(cid: string) {
-		return !!(await this._store.get(cid));
+		try {
+			return !!(await this._store.get(cid));
+		} catch (error: any) {
+			if (
+				typeof error?.code === "string" &&
+				error?.code?.indexOf("LEVEL_NOT_FOUND") !== -1
+			) {
+				return false;
+			}
+			throw error;
+		}
+	}
+
+	async hasMany(cids: string[]): Promise<boolean[]> {
+		const store = this._store as AnyStore & {
+			getMany?: (
+				keys: string[],
+			) => Promise<Array<Uint8Array | undefined>> | Array<Uint8Array | undefined>;
+			hasMany?: (keys: string[]) => Promise<boolean[]> | boolean[];
+		};
+		if (typeof store.hasMany === "function") {
+			return store.hasMany(cids);
+		}
+		if (typeof store.getMany === "function") {
+			const values = await store.getMany(cids);
+			return values.map((value) => value != null);
+		}
+
+		return Promise.all(cids.map((cid) => this.has(cid)));
 	}
 
 	async start(): Promise<void> {

package/src/interface.ts

@@ -1,10 +1,32 @@
 import type { MaybePromise } from "@peerbit/any-store-interface";
-import type { Blocks as IBlockStore } from "@peerbit/blocks-interface";
+import type { Blocks as IBlockStore, GetOptions } from "@peerbit/blocks-interface";
+import type { Block } from "multiformats/block";
 
 export type StoreStatus = MaybePromise<
 	"open" | "opening" | "closed" | "closing"
 >;
 export interface BlockStore extends IBlockStore {
+	putMany(
+		blocks: Array<Uint8Array | { block: Block<any, any, any, any>; cid: string }>,
+	): MaybePromise<string[]>;
+	putKnown(cid: string, bytes: Uint8Array): MaybePromise<string>;
+	putKnownMany(
+		blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
+	): MaybePromise<string[]>;
+	getMany(
+		cids: string[],
+		options?: GetOptions,
+	): MaybePromise<Array<Uint8Array | undefined>>;
+	hasMany(cids: string[]): MaybePromise<boolean[]>;
+	rmMany(cids: string[]): MaybePromise<number | void>;
+	getNativeLogBlockStoreHandle?(): unknown;
+	/**
+	 * Serialize a `BlockResponse` payload for a natively stored block without
+	 * materializing the block bytes as a JS value. Only implemented by stores
+	 * backed by the native log block store; `undefined` when the block is not
+	 * held natively.
+	 */
+	getBlockResponsePayload?(cid: string): Uint8Array | undefined;
 	start(): Promise<void>;
 	stop(): Promise<void>;
 	status(): StoreStatus;

package/src/libp2p.ts

@@ -1,16 +1,21 @@
 import { deserialize, serialize } from "@dao-xyz/borsh";
 import { createStore } from "@peerbit/any-store";
+import type { AnyStore } from "@peerbit/any-store-interface";
 import type { GetOptions, Blocks as IBlocks } from "@peerbit/blocks-interface";
 import { getPublicKeyFromPeerId, type PublicSignKey } from "@peerbit/crypto";
 import { DirectStream } from "@peerbit/stream";
-import { type DirectStreamComponents } from "@peerbit/stream";
+import {
+	type DirectStreamComponents,
+	type RustCoreStream,
+} from "@peerbit/stream";
 import {
 	createRequestTransportContext,
 	type DataMessage,
 	type WaitForPeersFn,
 } from "@peerbit/stream-interface";
+import type { Block } from "multiformats/block";
 import { AnyBlockStore } from "./any-blockstore.js";
-import { BlockMessage, RemoteBlocks } from "./remote.js";
+import { BlockMessage, BlockRequest, BlockResponse, RemoteBlocks } from "./remote.js";
 
 export type DirectBlockComponents = DirectStreamComponents;
 
@@ -23,6 +28,7 @@ export class DirectBlock extends DirectStream implements IBlocks {
 		components: DirectBlockComponents,
 		options?: {
 			directory?: string;
+			localStore?: AnyStore;
 			canRelayMessage?: boolean;
 			localTimeout?: number;
 			messageProcessingConcurrency?: number;
@@ -47,8 +53,20 @@ export class DirectBlock extends DirectStream implements IBlocks {
 						maxProvidersPerCid?: number;
 				  };
 			requeryOnReachable?: number;
+			/**
+			 * Run the block-exchange protocol on the native DirectStream core
+			 * (`@peerbit/network-rust`): codec, provider resolution and caches
+			 * execute in wasm, and natively stored blocks are served without
+			 * surfacing their bytes to JS. Defaults to the same rust-core mode
+			 * as the underlying DirectStream.
+			 */
+			rustCore?: RustCoreStream | false;
 		},
 	) {
+		if (options?.directory && options.localStore) {
+			throw new Error("DirectBlock options cannot include both directory and localStore");
+		}
+
 		super(components, ["/peerbit/direct-block/1.0.0"], {
 			messageProcessingConcurrency: options?.messageProcessingConcurrency || 10,
 			canRelayMessage: options?.canRelayMessage ?? true,
@@ -56,9 +74,29 @@ export class DirectBlock extends DirectStream implements IBlocks {
 				dialer: false,
 				pruner: false,
 			},
+			rustCore: options?.rustCore,
 		});
 
+		const blockExchange = this.rustCore?.blockExchange;
 		const defaultResolveProviders = () => {
+			if (blockExchange) {
+				const negotiated = [...this.peers.keys()];
+				const connected: string[] = [];
+				for (const conn of this.components.connectionManager.getConnections()) {
+					try {
+						connected.push(
+							getPublicKeyFromPeerId(conn.remotePeer).hashcode(),
+						);
+					} catch {
+						// ignore unexpected key types
+					}
+				}
+				return blockExchange.defaultProviderCandidates(
+					negotiated,
+					connected,
+					this.publicKeyHash,
+				);
+			}
 			const out: string[] = [];
 			const push = (hash?: string) => {
 				if (!hash) return;
@@ -89,8 +127,11 @@ export class DirectBlock extends DirectStream implements IBlocks {
 			return out;
 		};
 		this.remoteBlocks = new RemoteBlocks({
-			local: new AnyBlockStore(createStore(options?.directory)),
-			publish: (message, options) => this.publish(serialize(message), options),
+			local: new AnyBlockStore(
+				options?.localStore ?? createStore(options?.directory),
+			),
+			publish: (message, options) =>
+				this.publish(this.encodeBlockMessage(message), options),
 			localTimeout: options?.localTimeout || 1000,
 			messageProcessingConcurrency: options?.messageProcessingConcurrency || 10,
 			waitFor: this.waitFor.bind(this) as WaitForPeersFn,
@@ -101,30 +142,97 @@ export class DirectBlock extends DirectStream implements IBlocks {
 			onPut: options?.onPut,
 			providerCache: options?.providerCache,
 			requeryOnReachable: options?.requeryOnReachable,
+			rust: blockExchange
+				? {
+						exchange: blockExchange,
+						publishRaw: (payload, options) => this.publish(payload, options),
+					}
+				: undefined,
 		});
 
 		this.onDataFn = (data: CustomEvent<DataMessage>) => {
-			data.detail?.data?.length &&
-				data.detail?.data.length > 0 &&
-				this.remoteBlocks.onMessage(
-					deserialize(data.detail.data!, BlockMessage),
-					{
-						from: data.detail.header.signatures?.publicKeys[0]?.hashcode(),
-						transport: createRequestTransportContext(data.detail),
-					},
-				);
+			if (!(data.detail?.data?.length && data.detail.data.length > 0)) {
+				return;
+			}
+			let message: BlockMessage;
+			try {
+				message = this.decodeBlockMessage(data.detail.data!);
+			} catch {
+				// Drop undecodable frames instead of letting the throw escape
+				// the 'data' listener (an uncaughtException in Node). The TS
+				// borsh decoder tolerates malformed frames lossily and fails
+				// later inside handleFetchRequest, which is caught; the native
+				// decoder throws on e.g. non-UTF-8 cid bytes, so match the
+				// tolerant behaviour here.
+				return;
+			}
+			this.remoteBlocks.onMessage(message, {
+				from: data.detail.header.signatures?.publicKeys[0]?.hashcode(),
+				transport: createRequestTransportContext(data.detail),
+			});
 		};
 		this.onPeerConnectedFn = (evt: CustomEvent<PublicSignKey>) =>
 			this.remoteBlocks.onReachable(evt.detail);
 	}
 
-	async put(bytes: Uint8Array): Promise<string> {
+	private encodeBlockMessage(message: BlockRequest | BlockResponse): Uint8Array {
+		const blockExchange = this.rustCore?.blockExchange;
+		if (blockExchange) {
+			if (message instanceof BlockRequest) {
+				return blockExchange.encodeBlockRequest(message.cid);
+			}
+			if (message instanceof BlockResponse) {
+				return blockExchange.encodeBlockResponse(message.cid, message.bytes);
+			}
+		}
+		return serialize(message);
+	}
+
+	private decodeBlockMessage(bytes: Uint8Array): BlockMessage {
+		const blockExchange = this.rustCore?.blockExchange;
+		if (blockExchange) {
+			const decoded = blockExchange.decodeBlockMessage(bytes);
+			return decoded.type === "request"
+				? new BlockRequest(decoded.cid)
+				: new BlockResponse(decoded.cid, decoded.bytes);
+		}
+		return deserialize(bytes, BlockMessage);
+	}
+
+	getNativeLogBlockStoreHandle(): unknown {
+		return this.remoteBlocks.getNativeLogBlockStoreHandle();
+	}
+
+	async put(
+		bytes: Uint8Array | { block: Block<any, any, any, any>; cid: string },
+	): Promise<string> {
 		return this.remoteBlocks.put(bytes);
 	}
 
+	async putMany(
+		blocks: Array<
+			Uint8Array | { block: Block<any, any, any, any>; cid: string }
+		>,
+	): Promise<string[]> {
+		return this.remoteBlocks.putMany(blocks);
+	}
+
+	async putKnown(cid: string, bytes: Uint8Array): Promise<string> {
+		return this.remoteBlocks.putKnown(cid, bytes);
+	}
+
+	async putKnownMany(
+		blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
+	): Promise<string[]> {
+		return this.remoteBlocks.putKnownMany(blocks);
+	}
+
 	async has(cid: string) {
 		return this.remoteBlocks.has(cid);
 	}
+	async hasMany(cids: string[]): Promise<boolean[]> {
+		return this.remoteBlocks.hasMany(cids);
+	}
 	async get(
 		cid: string,
 		options?: GetOptions | undefined,
@@ -132,6 +240,13 @@ export class DirectBlock extends DirectStream implements IBlocks {
 		return this.remoteBlocks.get(cid, options);
 	}
 
+	async getMany(
+		cids: string[],
+		options?: GetOptions | undefined,
+	): Promise<Array<Uint8Array | undefined>> {
+		return this.remoteBlocks.getMany(cids, options);
+	}
+
 	hintProviders(cid: string, providers: string[]) {
 		this.remoteBlocks.hintProviders(cid, providers);
 	}
@@ -140,6 +255,10 @@ export class DirectBlock extends DirectStream implements IBlocks {
 		return this.remoteBlocks.rm(cid);
 	}
 
+	async rmMany(cids: string[]) {
+		return this.remoteBlocks.rmMany(cids);
+	}
+
 	async *iterator(): AsyncGenerator<[string, Uint8Array], void, void> {
 		for await (const [key, value] of this.remoteBlocks.iterator()) {
 			yield [key, value];