@peerbit/blocks 4.1.9 → 4.2.1

package/src/remote.ts

@@ -11,7 +11,12 @@ import {
 import { Cache } from "@peerbit/cache";
 import { PublicSignKey } from "@peerbit/crypto";
 import { logger as loggerFn } from "@peerbit/logger";
-import { type PublishOptions, dontThrowIfDeliveryError } from "@peerbit/stream";
+import {
+	type PublishOptions,
+	type RustBlockExchange,
+	type RustBlockProviderCache,
+	dontThrowIfDeliveryError,
+} from "@peerbit/stream";
 import {
 	type RequestTransportContext,
 	type PeerRefs,
@@ -24,7 +29,6 @@ import { AbortError } from "@peerbit/time";
 import { CID } from "multiformats";
 import { type Block } from "multiformats/block";
 import PQueue from "p-queue";
-import { AnyBlockStore } from "./any-blockstore.js";
 import type { BlockStore } from "./interface.js";
 
 export const logger = loggerFn("peerbit:transport:blocks");
@@ -72,22 +76,42 @@ type RemoteReadOptions = Exclude<GetOptions["remote"], boolean | undefined> & {
 	hasher?: any;
 };
 
+/**
+ * Shared shape of the TS eager-block cache (`Cache<Uint8Array>`) and the
+ * native-backed one (`RustEagerBlockCache`).
+ */
+type EagerBlockCache = {
+	add(cid: string, bytes: Uint8Array): void;
+	get(cid: string): Uint8Array | null | undefined;
+	del(cid: string): unknown;
+	clear(): void;
+};
+
 export class RemoteBlocks implements IBlocks {
 	localStore: BlockStore;
 
+	// Assigned in the constructor only when the local store supports it:
+	// callers feature-detect this method (the log's columnar raw-receive
+	// fast path), so it must not exist without a delegation target.
+	putKnownManyColumns?: (cids: string[], bytes: Uint8Array[]) => string[];
+
 	private _responseHandler?: (
 		data: BlockMessage,
 		context?: BlockMessageContext,
 	) => any;
 	private _resolvers: Map<string, (data: Uint8Array) => Promise<void>>;
-	private _blockCache?: Cache<Uint8Array>;
+	private _blockCache?: EagerBlockCache;
 	private _providerCache?: Cache<string[]>;
+	private _rustProviderCache?: RustBlockProviderCache;
+	private readonly rustExchange?: RustBlockExchange;
 	private readonly publicKeyHash: string;
 	private readonly maxProviderHintsPerCid: number;
 	private readonly maxRequeryOnReachable: number;
 
 	private _loadFetchQueue: PQueue;
 	private _readFromPeersPromises: Map<string, InFlightRead>;
+	private _deferredStoredNotificationCids?: Set<string>;
+	private _deferredStoredNotificationTimer?: ReturnType<typeof setTimeout>;
 	_open = false;
 	private _events: TypedEventEmitter<{
 		"peer:reachable": CustomEvent<PublicSignKey>;
@@ -97,7 +121,7 @@ export class RemoteBlocks implements IBlocks {
 
 	constructor(
 		readonly options: {
-			local: AnyBlockStore;
+			local: BlockStore;
 			localTimeout?: number;
 			messageProcessingConcurrency?: number;
 			publicKey: PublicSignKey;
@@ -158,25 +182,59 @@ export class RemoteBlocks implements IBlocks {
 				options: PublishOptions,
 			) => Promise<Uint8Array | undefined | void>;
 			waitFor: WaitForPeersFn;
+			/**
+			 * Native block-exchange components (rust-core mode). When set, the
+			 * provider caches/decisions and eager-block bookkeeping run in the
+			 * native core; `publishRaw` additionally enables serving natively
+			 * stored blocks as wasm-serialized payloads that never surface the
+			 * block bytes to JS.
+			 */
+			rust?: {
+				exchange: RustBlockExchange;
+				publishRaw?: (
+					payload: Uint8Array,
+					options: PublishOptions,
+				) => Promise<Uint8Array | undefined | void>;
+			};
 		},
 	) {
 		const localTimeout = options?.localTimeout || 1000;
 		this.publicKeyHash = options.publicKey.hashcode();
+		this.rustExchange = options.rust?.exchange;
 		this._loadFetchQueue = new PQueue({
 			concurrency: options?.messageProcessingConcurrency || 10,
 		});
 		this.localStore = options?.local;
+		const localPutKnownManyColumns = (
+			this.localStore as {
+				putKnownManyColumns?: (
+					cids: string[],
+					bytes: Uint8Array[],
+				) => string[];
+			}
+		)?.putKnownManyColumns;
+		if (typeof localPutKnownManyColumns === "function") {
+			this.putKnownManyColumns = (cids, bytes) => {
+				const stored = localPutKnownManyColumns.call(
+					this.localStore,
+					cids,
+					bytes,
+				);
+				void this.notifyPuts(stored);
+				return stored;
+			};
+		}
 		this._resolvers = new Map();
 		this._readFromPeersPromises = new Map();
-		this._blockCache = options?.eagerBlocks
-			? new Cache<Uint8Array>({
-					max:
-						typeof options.eagerBlocks === "boolean"
-							? 1e3
-							: (options.eagerBlocks.cacheSize ?? 1e3),
-					ttl: 1e4,
-				})
-			: undefined;
+		if (options?.eagerBlocks) {
+			const eagerBlocksMax =
+				typeof options.eagerBlocks === "boolean"
+					? 1e3
+					: (options.eagerBlocks.cacheSize ?? 1e3);
+			this._blockCache = this.rustExchange
+				? this.rustExchange.createEagerCache({ max: eagerBlocksMax, ttl: 1e4 })
+				: new Cache<Uint8Array>({ max: eagerBlocksMax, ttl: 1e4 });
+		}
 		type ProviderCacheOptions = {
 			maxEntries?: number;
 			ttlMs?: number;
@@ -188,12 +246,21 @@ export class RemoteBlocks implements IBlocks {
 				: typeof options.providerCache === "object"
 					? options.providerCache
 					: {};
-		this._providerCache = providerCache
-			? new Cache<string[]>({
+		if (providerCache) {
+			if (this.rustExchange) {
+				this._rustProviderCache = this.rustExchange.createProviderCache({
+					me: this.publicKeyHash,
+					maxEntries: providerCache.maxEntries ?? 2048,
+					ttlMs: providerCache.ttlMs ?? 10 * 60 * 1000,
+					maxProvidersPerCid: providerCache.maxProvidersPerCid ?? 8,
+				});
+			} else {
+				this._providerCache = new Cache<string[]>({
 					max: providerCache.maxEntries ?? 2048,
 					ttl: providerCache.ttlMs ?? 10 * 60 * 1000,
-				})
-			: undefined;
+				});
+			}
+		}
 		this.maxProviderHintsPerCid = providerCache?.maxProvidersPerCid ?? 8;
 		this.maxRequeryOnReachable = options.requeryOnReachable ?? 4;
 
@@ -234,11 +301,22 @@ export class RemoteBlocks implements IBlocks {
 		};
 	}
 
+	getNativeLogBlockStoreHandle(): unknown {
+		return this.localStore.getNativeLogBlockStoreHandle?.();
+	}
+
 	private normalizeProviderHints(
 		providers: string[] | undefined,
 		limit = this.maxProviderHintsPerCid || 8,
 	): string[] {
 		if (!providers || providers.length === 0) return [];
+		if (this.rustExchange) {
+			return this.rustExchange.normalizeProviderHints(
+				providers,
+				this.publicKeyHash,
+				limit,
+			);
+		}
 		const out: string[] = [];
 		for (const p of providers) {
 			if (!p) continue;
@@ -252,6 +330,10 @@ export class RemoteBlocks implements IBlocks {
 	}
 
 	private rememberProvider(cidString: string, providerHash: string) {
+		if (this._rustProviderCache) {
+			this._rustProviderCache.rememberProvider(cidString, providerHash);
+			return;
+		}
 		if (!this._providerCache) return;
 		if (!providerHash || providerHash === this.publicKeyHash) return;
 		const current = this._providerCache.get(cidString) ?? [];
@@ -266,13 +348,30 @@ export class RemoteBlocks implements IBlocks {
 	}
 
 	private rememberProviderHints(cidString: string, providers: string[]) {
+		if (this._rustProviderCache) {
+			this._rustProviderCache.rememberHints(cidString, providers);
+			return;
+		}
 		if (!this._providerCache) return;
 		const normalized = this.normalizeProviderHints(providers);
 		if (normalized.length === 0) return;
 		this._providerCache.add(cidString, normalized);
 	}
 
+	private getCachedProviders(cidString: string): string[] | undefined {
+		return this._rustProviderCache
+			? this._rustProviderCache.get(cidString)
+			: (this._providerCache?.get(cidString) ?? undefined);
+	}
+
 	private pickRequestBatch(providers: string[], attempt: number): string[] {
+		if (this.rustExchange) {
+			return this.rustExchange.pickRequestBatch(
+				providers,
+				this.publicKeyHash,
+				attempt,
+			);
+		}
 		if (providers.length <= 1) {
 			return providers;
 		}
@@ -293,7 +392,7 @@ export class RemoteBlocks implements IBlocks {
 		// 1. cached providers (from previous reads)
 		// 2. resolveProviders hook (e.g. program-level replicators, DHT, tracker)
 		const cached = this.normalizeProviderHints(
-			this._providerCache?.get(cidString) ?? undefined,
+			this.getCachedProviders(cidString),
 		);
 		if (!this.options.resolveProviders) return cached;
 		if (cached.length > 0 && !options?.refresh) return cached;
@@ -316,18 +415,147 @@ export class RemoteBlocks implements IBlocks {
 			throw new Error("Local store not set");
 		}
 		const cid = await this.localStore!.put(bytes);
+		await this.notifyPut(cid);
+		return cid;
+	}
+
+	async putMany(
+		blocks: Array<Uint8Array | { block: Block<any, any, any, any>; cid: string }>,
+	): Promise<string[]> {
+		if (!this.localStore) {
+			throw new Error("Local store not set");
+		}
+		const cids = await this.localStore.putMany(blocks);
+		await this.notifyPuts(cids);
+		return cids;
+	}
+
+	async putKnown(cid: string, bytes: Uint8Array): Promise<string> {
+		if (!this.localStore) {
+			throw new Error("Local store not set");
+		}
+		const storedCid = await this.localStore.putKnown(cid, bytes);
+		await this.notifyPut(storedCid);
+		return storedCid;
+	}
+
+	async putKnownMany(
+		blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
+	): Promise<string[]> {
+		if (!this.localStore) {
+			throw new Error("Local store not set");
+		}
+		const cids = await this.localStore.putKnownMany(blocks);
+		await this.notifyPuts(cids);
+		return cids;
+	}
+
+	hasNotifyStoredHook(): boolean {
+		return !!this.options.onPut;
+	}
+
+	notifyStored(cid: string): Promise<void> | void {
+		return this.notifyPut(cid);
+	}
+
+	notifyStoredMany(cids: string[]): Promise<void> | void {
+		return this.notifyPuts(cids);
+	}
+
+	notifyStoredDeferred(cid: string): void {
+		this.notifyStoredManyDeferred([cid]);
+	}
+
+	notifyStoredManyDeferred(cids: string[]): void {
+		if (!this.options.onPut || cids.length === 0) {
+			return;
+		}
+		let pending = this._deferredStoredNotificationCids;
+		if (!pending) {
+			pending = new Set();
+			this._deferredStoredNotificationCids = pending;
+		}
+		for (const cid of cids) {
+			if (cid) {
+				pending.add(cid);
+			}
+		}
+		if (pending.size === 0 || this._deferredStoredNotificationTimer) {
+			return;
+		}
+		this._deferredStoredNotificationTimer = setTimeout(() => {
+			this._deferredStoredNotificationTimer = undefined;
+			const flushed = this.flushDeferredStoredNotifications();
+			if (flushed && typeof flushed.catch === "function") {
+				flushed.catch((): void => undefined);
+			}
+		}, 0);
+	}
+
+	private flushDeferredStoredNotifications(): Promise<void> | void {
+		const pending = this._deferredStoredNotificationCids;
+		if (!pending || pending.size === 0) {
+			return;
+		}
+		this._deferredStoredNotificationCids = undefined;
+		const waits: Promise<void>[] = [];
+		for (const cid of pending) {
+			try {
+				const result = this.notifyStored(cid);
+				if (result && typeof result.then === "function") {
+					waits.push(result);
+				}
+			} catch {
+				// ignore best-effort hooks
+			}
+		}
+		if (waits.length > 0) {
+			return Promise.all(waits).then((): void => undefined);
+		}
+	}
+
+	private notifyPut(cid: string): Promise<void> | void {
+		const onPut = this.options.onPut;
+		if (!onPut) {
+			return;
+		}
 		try {
-			await this.options.onPut?.(cid);
+			const result = onPut(cid);
+			if (result && typeof result.then === "function") {
+				return result.catch((): void => undefined);
+			}
 		} catch {
 			// ignore best-effort hooks
 		}
-		return cid;
+	}
+
+	private notifyPuts(cids: string[]): Promise<void> | void {
+		const onPut = this.options.onPut;
+		if (!onPut || cids.length === 0) {
+			return;
+		}
+		if (cids.length === 1) {
+			return this.notifyPut(cids[0]!);
+		}
+		return Promise.all(
+			cids.map(async (cid) => {
+				try {
+					await onPut(cid);
+				} catch {
+					// ignore best-effort hooks
+				}
+			}),
+		).then((): void => undefined);
 	}
 
 	async has(cid: string) {
 		return this.localStore.has(cid);
 	}
 
+	async hasMany(cids: string[]): Promise<boolean[]> {
+		return this.localStore.hasMany(cids);
+	}
+
 	async get(
 		cid: string,
 		options?: GetOptions | undefined,
@@ -351,6 +579,16 @@ export class RemoteBlocks implements IBlocks {
 		return value;
 	}
 
+	async getMany(
+		cids: string[],
+		options?: GetOptions | undefined,
+	): Promise<Array<Uint8Array | undefined>> {
+		if (!options?.remote) {
+			return this.localStore.getMany(cids, options);
+		}
+		return Promise.all(cids.map((cid) => this.get(cid, options)));
+	}
+
 	hintProviders(cid: string, providers: string[]) {
 		const cidString = stringifyCid(cid);
 		this.rememberProviderHints(cidString, providers);
@@ -363,6 +601,10 @@ export class RemoteBlocks implements IBlocks {
 		await this.localStore?.rm(cid);
 	}
 
+	async rmMany(cids: string[]) {
+		await this.localStore?.rmMany(cids);
+	}
+
 	async *iterator(): AsyncGenerator<[string, Uint8Array], void, void> {
 		for await (const [key, value] of this.localStore.iterator()) {
 			yield [key, value];
@@ -396,6 +638,22 @@ export class RemoteBlocks implements IBlocks {
 			return;
 		}
 		const cid = stringifyCid(request.cid);
+		const publishRaw = this.options.rust?.publishRaw;
+		if (publishRaw) {
+			// Native-store-served response: the borsh BlockResponse payload is
+			// serialized inside wasm straight from the native log block store,
+			// so the block bytes never materialize as a JS value.
+			const payload = this.localStore.getBlockResponsePayload?.(cid);
+			if (payload) {
+				const responsePublishOptions = context?.transport
+					? context.transport.withResponseOptions({ to: [from] })
+					: { to: [from] };
+				await publishRaw(payload, responsePublishOptions).catch(
+					dontThrowIfDeliveryError,
+				);
+				return;
+			}
+		}
 		let bytes = await this.localStore.get(cid, {
 			remote: {
 				timeout: localTimeout,
@@ -459,7 +717,7 @@ export class RemoteBlocks implements IBlocks {
 		cidObject: CID,
 		options: RemoteReadOptions = {},
 	): Promise<Uint8Array | undefined> {
-		const codec = (codecCodes as any)[cidObject.code];
+		const codec = codecCodes[cidObject.code as keyof typeof codecCodes];
 
 		const tryDecode = async (bytes: Uint8Array) => {
 			const value = await checkDecodeBlock(cidObject, bytes, {
@@ -756,6 +1014,11 @@ export class RemoteBlocks implements IBlocks {
 
 		// Wait for processing request
 		this.closeController.abort();
+		if (this._deferredStoredNotificationTimer) {
+			clearTimeout(this._deferredStoredNotificationTimer);
+			this._deferredStoredNotificationTimer = undefined;
+		}
+		await this.flushDeferredStoredNotifications();
 		this._loadFetchQueue.clear();
 		await this._loadFetchQueue.onIdle(); // wait for pending
 		await this.localStore?.stop();
@@ -763,6 +1026,7 @@ export class RemoteBlocks implements IBlocks {
 		this._resolvers.clear();
 		this._blockCache?.clear();
 		this._providerCache?.clear();
+		this._rustProviderCache?.clear();
 		this._open = false;
 		// we dont cleanup subscription because we dont know if someone else is sbuscribing also
 	}