@peerbit/document 15.0.12 → 15.0.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/document",
-  "version": "15.0.12",
+  "version": "15.0.14",
   "description": "Document store implementation",
   "type": "module",
   "sideEffects": false,
@@ -62,19 +62,19 @@
     "uint8arrays": "^5.1.0",
     "@peerbit/cache": "3.1.1",
     "@peerbit/crypto": "3.1.6",
-    "@peerbit/document-interface": "3.2.65",
-    "@peerbit/indexer-interface": "3.0.11",
-    "@peerbit/indexer-simple": "1.2.15",
-    "@peerbit/indexer-sqlite3": "3.0.17",
-    "@peerbit/log": "6.2.22",
-    "@peerbit/indexer-cache": "0.2.14",
-    "@peerbit/program": "6.0.53",
-    "@peerbit/logger": "2.0.2",
+    "@peerbit/indexer-cache": "0.2.15",
+    "@peerbit/indexer-simple": "1.2.16",
+    "@peerbit/document-interface": "3.2.67",
+    "@peerbit/indexer-interface": "3.0.12",
+    "@peerbit/indexer-sqlite3": "3.0.18",
+    "@peerbit/log": "6.2.24",
     "@peerbit/pubsub": "5.4.2",
-    "@peerbit/rpc": "6.1.21",
+    "@peerbit/program": "6.0.55",
+    "@peerbit/logger": "2.0.2",
+    "@peerbit/rpc": "6.1.23",
     "@peerbit/stream-interface": "6.0.16",
     "@peerbit/time": "3.0.1",
-    "@peerbit/shared-log": "16.0.12"
+    "@peerbit/shared-log": "16.0.14"
   },
   "optionalDependencies": {
     "@peerbit/document-rust": "0.1.1"
@@ -87,8 +87,8 @@
     "uuid": "^11.1.1",
     "@peerbit/log-rust": "1.1.4",
     "@peerbit/native-backbone": "0.2.10",
-    "@peerbit/test-utils": "3.1.32",
-    "peerbit": "5.3.32"
+    "peerbit": "5.3.34",
+    "@peerbit/test-utils": "3.1.34"
   },
   "repository": {
     "type": "git",
@@ -102,6 +102,8 @@
   "scripts": {
     "clean": "aegir clean",
     "build": "aegir build --no-bundle",
+    "bench:ordered-remote-projection": "node --loader ts-node/esm ./benchmark/ordered-remote-projection.ts",
+    "bench:shared-fs-cold-join": "node --loader ts-node/esm ./benchmark/shared-fs-cold-join.ts",
     "test": "aegir test --target node",
     "test:document-rust-core": "npm run build && PEERBIT_SHARED_LOG_RUST_CORE=1 aegir test -t node --grep '^(?!.*(uses the validated local append path|uses the independent native prepared batch path|uses native shared-log planning for replicated target-none puts|uses commit-only local puts when coordinate persistence is deferred)).*(native conformance guard |operations basic |operations get |operations index |operations search |iterate sort |count approximate |query distribution |returnIndexed |caching |custom index |prefetch )' -- --forbid-pending",
     "lint": "aegir lint",

package/src/callback-detachment.ts

@@ -0,0 +1,308 @@
+import * as indexerTypes from "@peerbit/indexer-interface";
+import { Entry } from "@peerbit/log";
+import {
+	DeleteByStringKeyOperation,
+	DeleteOperation,
+	type Operation,
+	PutOperation,
+	PutWithKeyOperation,
+} from "./operation.js";
+
+/**
+ * Borsh byte fields are input-buffer views. Clone only fields that can expose
+ * verified operation bytes while retaining the operation's exact prototype,
+ * subclass fields, symbols, and property descriptors.
+ */
+export const detachOperationBytes = <T extends Operation>(operation: T): T => {
+	const cloneWithField = <V extends object>(
+		value: V,
+		key: PropertyKey,
+		replacement: unknown,
+	): V => {
+		const descriptors = Object.getOwnPropertyDescriptors(value) as Record<
+			PropertyKey,
+			PropertyDescriptor
+		>;
+		descriptors[key] = {
+			...descriptors[key],
+			value: replacement,
+		};
+		const clone = Object.create(Object.getPrototypeOf(value), descriptors) as V;
+		if (!Object.isExtensible(value)) {
+			Object.preventExtensions(clone);
+		}
+		return clone;
+	};
+	if (operation instanceof PutWithKeyOperation) {
+		return cloneWithField(
+			operation,
+			"data",
+			new Uint8Array(operation.data),
+		) as T;
+	}
+	if (operation instanceof PutOperation) {
+		return cloneWithField(
+			operation,
+			"data",
+			new Uint8Array(operation.data),
+		) as T;
+	}
+	if (operation instanceof DeleteOperation) {
+		const key = cloneWithField(
+			operation.key,
+			"key",
+			operation.key instanceof indexerTypes.Uint8ArrayKey
+				? new Uint8Array(operation.key.key)
+				: operation.key.key,
+		);
+		return cloneWithField(operation, "key", key) as T;
+	}
+	if (operation instanceof DeleteByStringKeyOperation) {
+		return cloneWithField(operation, "key", operation.key) as T;
+	}
+	return operation;
+};
+
+type LazyCallbackProperty = "operation" | "entry";
+type LazyCallbackState = {
+	operation?: Operation;
+	entry?: Entry<any>;
+	operationPending: boolean;
+	entryPending: boolean;
+};
+
+const lazyCallbackStates = new WeakMap<object, LazyCallbackState>();
+
+const isLazyCallbackProperty = (
+	property: PropertyKey,
+): property is LazyCallbackProperty =>
+	property === "operation" || property === "entry";
+
+const settleCallbackProperty = (
+	target: object,
+	state: LazyCallbackState,
+	property: LazyCallbackProperty,
+): void => {
+	state[`${property}Pending`] = false;
+	state[property] = undefined;
+	if (!state.operationPending && !state.entryPending) {
+		lazyCallbackStates.delete(target);
+	}
+};
+
+const materializeCallbackProperty = (
+	target: object,
+	property: LazyCallbackProperty,
+): void => {
+	const state = lazyCallbackStates.get(target);
+	if (!state || !state[`${property}Pending`]) {
+		return;
+	}
+	const value =
+		property === "operation"
+			? detachOperationBytes(state.operation!)
+			: detachEntryPayloadForCallback(state.entry!);
+	Reflect.set(target, property, value, target);
+	settleCallbackProperty(target, state, property);
+};
+
+const lazyCallbackHandler: ProxyHandler<any> = {
+	get(target, property, receiver) {
+		if (isLazyCallbackProperty(property)) {
+			materializeCallbackProperty(target, property);
+		}
+		return Reflect.get(target, property, receiver);
+	},
+	set(target, property, value) {
+		const updated = Reflect.set(target, property, value, target);
+		if (updated && isLazyCallbackProperty(property)) {
+			const state = lazyCallbackStates.get(target);
+			if (state) {
+				settleCallbackProperty(target, state, property);
+			}
+		}
+		return updated;
+	},
+	deleteProperty(target, property) {
+		const deleted = Reflect.deleteProperty(target, property);
+		if (deleted && isLazyCallbackProperty(property)) {
+			const state = lazyCallbackStates.get(target);
+			if (state) {
+				settleCallbackProperty(target, state, property);
+			}
+		}
+		return deleted;
+	},
+	defineProperty(target, property, descriptor) {
+		if (isLazyCallbackProperty(property)) {
+			const replacesValue =
+				"value" in descriptor || "get" in descriptor || "set" in descriptor;
+			if (!replacesValue) {
+				materializeCallbackProperty(target, property);
+			}
+			const defined = Reflect.defineProperty(target, property, descriptor);
+			if (defined && replacesValue) {
+				const state = lazyCallbackStates.get(target);
+				if (state) {
+					settleCallbackProperty(target, state, property);
+				}
+			}
+			return defined;
+		}
+		return Reflect.defineProperty(target, property, descriptor);
+	},
+	getOwnPropertyDescriptor(target, property) {
+		if (isLazyCallbackProperty(property)) {
+			materializeCallbackProperty(target, property);
+		}
+		return Reflect.getOwnPropertyDescriptor(target, property);
+	},
+};
+
+export const detachCanPerformCallbackProperties = <
+	T extends { operation: Operation; entry: Entry<any> },
+>(
+	properties: T,
+): T => {
+	lazyCallbackStates.set(properties, {
+		operation: properties.operation,
+		entry: properties.entry,
+		operationPending: true,
+		entryPending: true,
+	});
+	Reflect.set(properties, "operation", undefined, properties);
+	Reflect.set(properties, "entry", undefined, properties);
+	return new Proxy(properties, lazyCallbackHandler);
+};
+
+/**
+ * Return an Entry view whose public payload accessors cannot expose the
+ * verified payload buffer to application callbacks. The byte copy is lazy:
+ * callbacks that only inspect hashes, clocks, or signers allocate nothing.
+ */
+const detachedCallbackEntries = new WeakSet<object>();
+
+export const detachEntryPayloadForCallback = <T>(entry: Entry<T>): Entry<T> => {
+	if (detachedCallbackEntries.has(entry)) {
+		return entry;
+	}
+	let detachedPayload: any;
+	let detachedMaterialized: Entry<T> | undefined;
+	let callbackEntry: Entry<T>;
+	const boundMethods = new Map<
+		PropertyKey,
+		{ source: Function; bound: Function }
+	>();
+	const copyPayload = (payload: any) => {
+		if (detachedPayload) {
+			return detachedPayload;
+		}
+		const data = new Uint8Array(payload.data);
+		const descriptors = Object.getOwnPropertyDescriptors(payload) as Record<
+			PropertyKey,
+			PropertyDescriptor
+		>;
+		descriptors.data = {
+			...descriptors.data,
+			value: data,
+		};
+		descriptors._value = {
+			configurable: true,
+			enumerable: false,
+			writable: true,
+			...descriptors._value,
+			value: payload.isDecoded ? payload.encoding.decoder(data) : undefined,
+		};
+		detachedPayload = Object.create(
+			Object.getPrototypeOf(payload),
+			descriptors,
+		);
+		if (!Object.isExtensible(payload)) {
+			Object.preventExtensions(detachedPayload);
+		}
+		return detachedPayload;
+	};
+	const getPayload = async () => {
+		const entryWithPayloadMethod = entry as Entry<T> & {
+			getPayload?: () => unknown;
+		};
+		if (typeof entryWithPayloadMethod.getPayload === "function") {
+			return copyPayload(await entryWithPayloadMethod.getPayload.call(entry));
+		}
+		const materialized = entry.toMaterialized();
+		if (materialized !== entry) {
+			const materializedWithPayloadMethod = materialized as Entry<T> & {
+				getPayload?: () => unknown;
+			};
+			if (typeof materializedWithPayloadMethod.getPayload === "function") {
+				return copyPayload(
+					await materializedWithPayloadMethod.getPayload.call(materialized),
+				);
+			}
+			return copyPayload(materialized.payload);
+		}
+		return copyPayload(entry.payload);
+	};
+	const getPayloadValue = async () => {
+		const payload = await getPayload();
+		return payload.isDecoded ? payload.value : payload.getValue();
+	};
+	callbackEntry = new Proxy(entry, {
+		get(target, property) {
+			if (property === "payload") {
+				return copyPayload(entry.payload);
+			}
+			if (
+				property === "getPayload" &&
+				typeof (entry as Entry<T> & { getPayload?: unknown }).getPayload ===
+					"function"
+			) {
+				return getPayload;
+			}
+			if (property === "getPayloadValue") {
+				return getPayloadValue;
+			}
+			if (property === "getStorageBytes") {
+				return () => new Uint8Array(entry.getStorageBytes());
+			}
+			if (property === "valueOf") {
+				return () => callbackEntry;
+			}
+			if (property === "toMaterialized") {
+				return () => {
+					const materialized = entry.toMaterialized();
+					if (materialized === entry) {
+						return callbackEntry;
+					}
+					return (detachedMaterialized ??=
+						detachEntryPayloadForCallback(materialized));
+				};
+			}
+			if (property === "toSignable") {
+				return () => detachEntryPayloadForCallback(entry.toSignable());
+			}
+			if (property === "init") {
+				return (properties: Parameters<Entry<T>["init"]>[0]) => {
+					entry.init(properties);
+					return callbackEntry;
+				};
+			}
+			const value = Reflect.get(target, property, target);
+			if (property === "constructor" || typeof value !== "function") {
+				return value;
+			}
+			const existing = boundMethods.get(property);
+			if (existing && existing.source === value) {
+				return existing.bound;
+			}
+			const bound = value.bind(target);
+			boundMethods.set(property, { source: value, bound });
+			return bound;
+		},
+		set(target, property, value) {
+			return Reflect.set(target, property, value, target);
+		},
+	});
+	detachedCallbackEntries.add(callbackEntry);
+	return callbackEntry;
+};

package/src/domain.ts

@@ -9,6 +9,7 @@ import {
 	type ReplicationDomain,
 	type SharedLog,
 } from "@peerbit/shared-log";
+import { detachEntryPayloadForCallback } from "./callback-detachment.js";
 import { type Operation, isPutOperation } from "./operation.js";
 import type { DocumentIndex } from "./search.js";
 
@@ -93,8 +94,13 @@ export const createDocumentDomain =
 	): ((db: DB) => CustomDocumentDomain<InferR<DB>>) =>
 	(db: DB) => {
 		let maxValue = args.resolution === "u32" ? MAX_U32 : MAX_U64;
+		const detachEntry = (
+			entry: ShallowEntry | Entry<Operation> | EntryReplicated<any>,
+		) =>
+			entry instanceof Entry ? detachEntryPayloadForCallback(entry) : entry;
 		let fromEntry = (args as FromEntry<InferR<DB>>).fromEntry
-			? (args as FromEntry<InferR<DB>>).fromEntry!
+			? (entry: ShallowEntry | Entry<Operation> | EntryReplicated<any>) =>
+					(args as FromEntry<InferR<DB>>).fromEntry!(detachEntry(entry))
 			: async (
 					entry: ShallowEntry | Entry<Operation> | EntryReplicated<any>,
 				) => {
@@ -106,11 +112,13 @@ export const createDocumentDomain =
 					if (!item) {
 						logger.error("Item not found");
 					} else if (isPutOperation(item)) {
-						document = db.index.valueEncoding.decoder(item.data);
+						document = db.index.valueEncoding.decoder(
+							new Uint8Array(item.data),
+						);
 					}
 					return (args as FromValue<any, any>).fromValue!(
 						document,
-						entry,
+						detachEntry(entry),
 					) as NumberFromType<InferR<DB>>;
 				};
 		return {