@peerbit/document 15.0.13 → 15.0.15

package/src/domain.ts

@@ -9,6 +9,7 @@ import {
 	type ReplicationDomain,
 	type SharedLog,
 } from "@peerbit/shared-log";
+import { detachEntryForCallback } from "./callback-detachment.js";
 import { type Operation, isPutOperation } from "./operation.js";
 import type { DocumentIndex } from "./search.js";
 
@@ -93,8 +94,12 @@ export const createDocumentDomain =
 	): ((db: DB) => CustomDocumentDomain<InferR<DB>>) =>
 	(db: DB) => {
 		let maxValue = args.resolution === "u32" ? MAX_U32 : MAX_U64;
+		const detachEntry = (
+			entry: ShallowEntry | Entry<Operation> | EntryReplicated<any>,
+		) => detachEntryForCallback(entry);
 		let fromEntry = (args as FromEntry<InferR<DB>>).fromEntry
-			? (args as FromEntry<InferR<DB>>).fromEntry!
+			? (entry: ShallowEntry | Entry<Operation> | EntryReplicated<any>) =>
+					(args as FromEntry<InferR<DB>>).fromEntry!(detachEntry(entry))
 			: async (
 					entry: ShallowEntry | Entry<Operation> | EntryReplicated<any>,
 				) => {
@@ -106,11 +111,13 @@ export const createDocumentDomain =
 					if (!item) {
 						logger.error("Item not found");
 					} else if (isPutOperation(item)) {
-						document = db.index.valueEncoding.decoder(item.data);
+						document = db.index.valueEncoding.decoder(
+							new Uint8Array(item.data),
+						);
 					}
 					return (args as FromValue<any, any>).fromValue!(
 						document,
-						entry,
+						detachEntry(entry),
 					) as NumberFromType<InferR<DB>>;
 				};
 		return {

package/src/program.ts

@@ -1,6 +1,7 @@
 import {
 	type AbstractType,
 	BorshError,
+	deserialize,
 	field,
 	serialize,
 	variant,
@@ -44,6 +45,11 @@ import {
 	SharedLog,
 	type SharedLogOptions,
 } from "@peerbit/shared-log";
+import {
+	detachCanPerformCallbackProperties,
+	detachEntryForCallback,
+	detachEntryPayloadForCallback,
+} from "./callback-detachment.js";
 import { MAX_BATCH_SIZE } from "./constants.js";
 import type { CustomDocumentDomain } from "./domain.js";
 import type { DocumentEvents, DocumentsChange } from "./events.js";
@@ -1088,12 +1094,23 @@ export class Documents<
 	private _documentChangeListenerTrackingInitialized = false;
 	private _documentBackend!: DocumentBackend<T>;
 	private _canAppendDecodedDocuments = new WeakMap<PutOperation, T>();
+	private acceptPutOperation(
+		operation: PutOperation,
+		document: T | undefined,
+		hasDocument: boolean,
+	): PutOperation {
+		if (hasDocument) {
+			this._canAppendDecodedDocuments.set(operation, document!);
+		}
+		return operation;
+	}
 	private _nativeDocumentIdExtractionPlan?: SimpleDocumentFieldExtractionPlan;
 	private _nativeDocumentFieldExtractionPlans?: Map<
 		string,
 		SimpleDocumentFieldExtractionPlan | undefined
 	>;
 	private _hasLogTrim = false;
+	private _hasCustomIdResolver = false;
 	private idResolver!: (any: any) => indexerTypes.Ideable;
 	private domain?: CustomDocumentDomain<InferR<D>>;
 	private strictHistory: boolean;
@@ -1131,6 +1148,99 @@ export class Documents<
 				);
 	}
 
+	private detachDomainEntryCallback(
+		domain: CustomDocumentDomain<InferR<D>>,
+	): CustomDocumentDomain<InferR<D>> {
+		const boundMethods = new Map<
+			PropertyKey,
+			{ source: Function; bound: Function }
+		>();
+		const detachedFromEntry = (
+			entry: Parameters<CustomDocumentDomain<InferR<D>>["fromEntry"]>[0],
+		) => {
+			const fromEntry = Reflect.get(
+				domain,
+				"fromEntry",
+				domain,
+			) as CustomDocumentDomain<InferR<D>>["fromEntry"];
+			return Reflect.apply(fromEntry, domain, [detachEntryForCallback(entry)]);
+		};
+		const facade = Object.create(
+			Object.getPrototypeOf(domain),
+		) as CustomDocumentDomain<InferR<D>>;
+		let callbackDomain: CustomDocumentDomain<InferR<D>>;
+		callbackDomain = new Proxy(facade, {
+			get(_target, property) {
+				if (property === "fromEntry") {
+					return detachedFromEntry;
+				}
+				if (property === "valueOf") {
+					return () => callbackDomain;
+				}
+				const value = Reflect.get(domain, property, domain);
+				if (property === "constructor" || typeof value !== "function") {
+					return value;
+				}
+				const existing = boundMethods.get(property);
+				if (existing && existing.source === value) {
+					return existing.bound;
+				}
+				const bound = value.bind(domain);
+				boundMethods.set(property, { source: value, bound });
+				return bound;
+			},
+			set(_target, property, value) {
+				return Reflect.set(domain, property, value, domain);
+			},
+			has(_target, property) {
+				return Reflect.has(domain, property);
+			},
+			ownKeys() {
+				return Reflect.ownKeys(domain);
+			},
+			getOwnPropertyDescriptor(_target, property) {
+				const descriptor = Reflect.getOwnPropertyDescriptor(domain, property);
+				if (!descriptor) {
+					return undefined;
+				}
+				if (property === "fromEntry") {
+					return "value" in descriptor
+						? {
+								...descriptor,
+								value: detachedFromEntry,
+								configurable: true,
+							}
+						: {
+								...descriptor,
+								get: () => detachedFromEntry,
+								configurable: true,
+							};
+				}
+				return { ...descriptor, configurable: true };
+			},
+		});
+		return callbackDomain;
+	}
+
+	private detachTrimEntryCallback(
+		option: TrimOptions | undefined,
+	): TrimOptions | undefined {
+		const filter = option?.filter;
+		if (!option || !filter?.canTrim) {
+			return option;
+		}
+		const canTrim = filter.canTrim;
+		return {
+			...option,
+			filter: {
+				...filter,
+				canTrim: function (this: unknown, entry: ShallowEntry) {
+					return Reflect.apply(canTrim, this, [detachEntryForCallback(entry)]);
+				},
+			},
+		} as TrimOptions;
+	}
+
 	private createNativeDocumentBackendContext(): NativeDocumentBackendContext<
 		T,
 		I
@@ -2356,6 +2466,7 @@ export class Documents<
 				: (obj: any) =>
 						indexerTypes.extractFieldValue(obj, idProperty as string[]));
 
+		this._hasCustomIdResolver = options.id != null;
 		this.idResolver = idResolver;
 		this.strictHistory = options.strictHistory ?? false;
 		this._hasLogTrim = options.log?.trim != null;
@@ -2404,7 +2515,9 @@ export class Documents<
 		// 7 -> log v9) is retired; the rejection at the top of open() fires for
 		// any defined value before this point.
 
-		this.domain = options.domain?.(this);
+		this.domain = options.domain
+			? this.detachDomainEntryCallback(options.domain(this))
+			: undefined;
 
 		let keepFunction:
 			| ((
@@ -2449,7 +2562,12 @@ export class Documents<
 				return false;
 			};
 		} else {
-			keepFunction = options?.keep;
+			const keep = options?.keep;
+			keepFunction = keep
+				? function (this: unknown, entry) {
+						return Reflect.apply(keep, this, [detachEntryForCallback(entry)]);
+					}
+				: undefined;
 		}
 
 		await this.log.open({
@@ -2457,7 +2575,7 @@ export class Documents<
 			canReplicate: options?.canReplicate,
 			canAppend: this.canAppend.bind(this),
 			onChange: this.handleChanges.bind(this),
-			trim: options?.log?.trim,
+			trim: this.detachTrimEntryCallback(options?.log?.trim),
 			appendDurability: options?.appendDurability,
 			nativeBackbone: options?.nativeBackbone,
 			nativeGraph: options?.nativeGraph,
@@ -2473,7 +2591,10 @@ export class Documents<
 			distributionDebounceTime: options?.distributionDebounceTime,
 			strictFullReplicaFallback: false,
 			domain: options?.domain
-				? () => options.domain!(this) as unknown as D
+				? () =>
+						this.detachDomainEntryCallback(
+							options.domain!(this),
+						) as unknown as D
 				: undefined,
 			eagerBlocks: options?.eagerBlocks,
 			fanout: options?.fanout,
@@ -2565,7 +2686,7 @@ export class Documents<
 		}
 
 		try {
-			let operation: PutOperation | DeleteOperation = l0;
+			const operation: PutOperation | DeleteOperation = l0;
 			if (this._optionCanPerform) {
 				if (this._optionCanPerformNativePolicy && this.isNativeMode()) {
 					return this.nativeCanPerformAllowsAppend(
@@ -2575,22 +2696,22 @@ export class Documents<
 						reference?.document,
 					);
 				}
-				let document: T | undefined = reference?.document;
-				if (!document) {
-					if (isPutOperation(l0)) {
-						document =
-							this._canAppendDecodedDocuments.get(l0) ??
-							this._index.valueEncoding.decoder(l0.data);
-						if (!document) {
-							return false;
-						}
-					} else if (isDeleteOperation(l0)) {
-						// Nothing to do here by default.
-						// Checking if the document exists is not necessary since it
-						// might already be deleted.
-					} else {
-						throw new Error("Unsupported operation");
+				let document: T | undefined;
+				if (isPutOperation(l0)) {
+					const cachedDocument = this._canAppendDecodedDocuments.get(l0);
+					this._canAppendDecodedDocuments.delete(l0);
+					document =
+						cachedDocument ??
+						this._index.valueEncoding.decoder(new Uint8Array(l0.data));
+					if (!document) {
+						return false;
 					}
+				} else if (isDeleteOperation(l0)) {
+					// Nothing to do here by default.
+					// Checking if the document exists is not necessary since it
+					// might already be deleted.
+				} else {
+					throw new Error("Unsupported operation");
 				}
 				const previousEntries =
 					this._optionCanPerformNativePolicy &&
@@ -2598,7 +2719,9 @@ export class Documents<
 					canPerformPolicyNeedsPreviousEntries(
 						this._optionCanPerformNativePolicy,
 					)
-						? await this.resolveCanPerformPreviousEntries(entry)
+						? (await this.resolveCanPerformPreviousEntries(entry)).map(
+								detachEntryPayloadForCallback,
+							)
 						: undefined;
 				const deleteValue =
 					this._optionCanPerformNativePolicy &&
@@ -2610,20 +2733,22 @@ export class Documents<
 						: undefined;
 				if (
 					!(await this._optionCanPerform(
-						isPutOperation(operation)
-							? {
-									type: "put",
-									value: document!,
-									operation,
-									entry: entry as unknown as Entry<PutOperation>,
-									previousEntries,
-								}
-							: {
-									type: "delete",
-									value: deleteValue,
-									operation,
-									entry: entry as unknown as Entry<DeleteOperation>,
-								},
+						detachCanPerformCallbackProperties(
+							isPutOperation(operation)
+								? {
+										type: "put",
+										value: document!,
+										operation,
+										entry: entry as unknown as Entry<PutOperation>,
+										previousEntries,
+									}
+								: {
+										type: "delete",
+										value: deleteValue,
+										operation,
+										entry: entry as unknown as Entry<DeleteOperation>,
+									},
+						),
 					))
 				) {
 					return false;
@@ -2712,6 +2837,12 @@ export class Documents<
 		return entries;
 	}
 
+	private detachDocumentValueForCallback(document: T): T {
+		return this._index.valueEncoding.decoder(
+			this._index.valueEncoding.encoder(document),
+		);
+	}
+
 	private async resolveCanPerformDeleteValue(
 		operation: DeleteOperation,
 		options?: { allowEntryFallback?: boolean },
@@ -2728,12 +2859,15 @@ export class Documents<
 		const indexedDocument =
 			await this.getLocalIdentityDocumentByHead(existingHead);
 		if (indexedDocument) {
-			return indexedDocument;
+			return this.detachDocumentValueForCallback(indexedDocument);
 		}
 		const indexedPolicyDocument =
 			await this.getLocalIndexedDocumentForNativeDeletePolicy(key);
 		if (indexedPolicyDocument) {
-			return indexedPolicyDocument;
+			return deserialize(
+				serialize(indexedPolicyDocument),
+				this._index.indexedType,
+			) as unknown as T;
 		}
 		if (options?.allowEntryFallback === false) {
 			return;
@@ -2745,7 +2879,9 @@ export class Documents<
 		if (!isPutOperation(existingOperation)) {
 			return;
 		}
-		return this._index.valueEncoding.decoder(existingOperation.data);
+		return this._index.valueEncoding.decoder(
+			new Uint8Array(existingOperation.data),
+		);
 	}
 
 	protected async _canAppend(
@@ -2796,17 +2932,32 @@ export class Documents<
 			if (isPutOperation(operation)) {
 				// check nexts
 				const putOperation = operation as PutOperation;
+				let decodedDocumentForCanPerform: T | undefined;
+				let hasDecodedDocumentForCanPerform = false;
 				let keyValue: indexerTypes.Ideable | undefined;
 				if (reference?.document) {
-					keyValue = this.idResolver(reference.document);
+					keyValue = this.idResolver(
+						this._hasCustomIdResolver
+							? this.index.valueEncoding.decoder(
+									new Uint8Array(putOperation.data),
+								)
+							: reference.document,
+					);
 				} else {
 					keyValue = await this.getNativeDocumentIdFromPutOperation(putOperation);
 					if (keyValue == null) {
 						if (this.isNativeMode()) {
 							return false;
 						}
-						const value = this.index.valueEncoding.decoder(putOperation.data);
-						this._canAppendDecodedDocuments.set(putOperation, value);
+						const value = this.index.valueEncoding.decoder(
+							this._hasCustomIdResolver || this._optionCanPerform
+								? new Uint8Array(putOperation.data)
+								: putOperation.data,
+						);
+						if (this._optionCanPerform && !this._hasCustomIdResolver) {
+							decodedDocumentForCanPerform = value;
+							hasDecodedDocumentForCanPerform = true;
+						}
 						keyValue = this.idResolver(value);
 					}
 				}
@@ -2843,7 +2994,11 @@ export class Documents<
 							return false; // can not append to immutable document
 						}
 
-						return putOperation;
+						return this.acceptPutOperation(
+							putOperation,
+							decodedDocumentForCanPerform,
+							hasDecodedDocumentForCanPerform,
+						);
 					} else {
 						if (this.strictHistory) {
 							// make sure that the next pointer exist and points to the existing documents
@@ -2851,7 +3006,11 @@ export class Documents<
 								return false;
 							}
 							if (entry.meta.next[0] === existingContext.head) {
-								return putOperation;
+								return this.acceptPutOperation(
+									putOperation,
+									decodedDocumentForCanPerform,
+									hasDecodedDocumentForCanPerform,
+								);
 							}
 
 							const prevEntry = await this.log.log.entryIndex.get(
@@ -2866,14 +3025,29 @@ export class Documents<
 							}
 							const referenceHistoryCorrectly =
 								await pointsToHistory(prevEntry);
-							return referenceHistoryCorrectly ? putOperation : false;
+							return referenceHistoryCorrectly
+								? this.acceptPutOperation(
+										putOperation,
+										decodedDocumentForCanPerform,
+										hasDecodedDocumentForCanPerform,
+									)
+								: false;
 						} else {
-							return putOperation;
+							return this.acceptPutOperation(
+								putOperation,
+								decodedDocumentForCanPerform,
+								hasDecodedDocumentForCanPerform,
+							);
 						}
 					}
 				} else {
 					// Keep existing behavior: next pointers may express document dependencies.
 				}
+				return this.acceptPutOperation(
+					putOperation,
+					decodedDocumentForCanPerform,
+					hasDecodedDocumentForCanPerform,
+				);
 			} else if (isDeleteOperation(operation)) {
 				if (entry.meta.next.length !== 1) {
 					return false;
@@ -2924,7 +3098,6 @@ export class Documents<
 				throw new Error("Unsupported operation");
 			}
 
-			return operation;
 		} catch (error) {
 			if (error instanceof AccessError) {
 				return false; // we cant index because we can not decrypt
@@ -5459,14 +5632,25 @@ export class Documents<
 						}
 					}
 					let value =
-						(isReferencedAppendEntry && reference?.document) ||
-						this.index.valueEncoding.decoder(payload.data);
+						isReferencedAppendEntry && reference?.document
+							? reference.document
+							: this.index.valueEncoding.decoder(
+									new Uint8Array(payload.data),
+								);
 
 					// get index key from value
 					const key =
 						isReferencedAppendEntry && reference?.key
 							? reference.key
-							: indexerTypes.toId(this.idResolver(value));
+							: indexerTypes.toId(
+									this.idResolver(
+										this._hasCustomIdResolver
+											? this.index.valueEncoding.decoder(
+													new Uint8Array(payload.data),
+												)
+											: value,
+									),
+								);
 
 					// document is already updated with more recent entry
 					if (modified.has(key.primitive)) {

package/src/search.ts

@@ -44,6 +44,7 @@ import { AbortError, TimeoutError, waitFor } from "@peerbit/time";
 import pDefer, { type DeferredPromise } from "p-defer";
 import { concat, equals, fromString } from "uint8arrays";
 import { copySerialization } from "./borsh.js";
+import { detachPublicKeysForCallback } from "./callback-detachment.js";
 import { MAX_BATCH_SIZE } from "./constants.js";
 import type { DocumentEvents, DocumentsChange } from "./events.js";
 import type { QueryPredictor } from "./most-common-query-predictor.js";
@@ -74,6 +75,7 @@ import {
 	canPrepareDocumentTransformWithAppendFacts,
 	documentTransformPreservesFieldPath,
 	getDocumentTransformDescriptor,
+	isBuiltInDocumentTransformer,
 } from "./transform.js";
 
 const WARNING_WHEN_ITERATING_FOR_MORE_THAN = 1e5;
@@ -1050,6 +1052,7 @@ export class DocumentIndex<
 	// transform options
 	transformer: Transformer<T, I>;
 	private transformerIsIdentity = false;
+	private detachTransformerFacts = false;
 	private nativeTransformDescriptor?: DocumentTransformDescriptor;
 	private nativeTransformProjectionPlan?: SimpleDocumentProjectionPlan;
 	private nativeBackboneDocumentProjection?: NativeBackboneDocumentProjection;
@@ -1631,6 +1634,9 @@ export class DocumentIndex<
 		this.nativeTransformDescriptor = hasTransformFunction
 			? getDocumentTransformDescriptor(transformOptions.transform)
 			: undefined;
+		this.detachTransformerFacts =
+			hasTransformFunction &&
+			!isBuiltInDocumentTransformer(transformOptions.transform);
 		this.nativeTransformProjectionPlan = createSimpleProjectionPlan(
 			getSchema(this.documentType),
 			indexedSchema,
@@ -1827,6 +1833,36 @@ export class DocumentIndex<
 		);
 	}
 
+	private transformWithDetachedFacts(
+		value: T,
+		context: types.Context,
+		facts?: DocumentTransformFacts,
+	): MaybePromise<I> {
+		let callbackFacts = facts;
+		if (facts?.entryPublicKeys && this.detachTransformerFacts) {
+			const sourceKeys = facts.entryPublicKeys;
+			let callbackKeys: readonly PublicSignKey[] | undefined;
+			let resolved = false;
+			callbackFacts = { ...facts };
+			Object.defineProperty(callbackFacts, "entryPublicKeys", {
+				configurable: true,
+				enumerable: true,
+				get: () => {
+					if (!resolved) {
+						callbackKeys = detachPublicKeysForCallback(sourceKeys);
+						resolved = true;
+					}
+					return callbackKeys;
+				},
+				set: (value: readonly PublicSignKey[] | undefined) => {
+					callbackKeys = value;
+					resolved = true;
+				},
+			});
+		}
+		return this.transformer(value, context, callbackFacts);
+	}
+
 	private prepareNativeBackboneDocumentIndexCommit(
 		value: T,
 		encodedDocument: Uint8Array,
@@ -1851,7 +1887,7 @@ export class DocumentIndex<
 				},
 				getIndexable: () => {
 					if (!hasCached) {
-						const transformed = this.transformer(
+						const transformed = this.transformWithDetachedFacts(
 							value,
 							projectionContext as types.Context,
 							transformFacts,
@@ -1878,7 +1914,7 @@ export class DocumentIndex<
 		) {
 			return;
 		}
-		const transformed = this.transformer(
+		const transformed = this.transformWithDetachedFacts(
 			value,
 			undefined as unknown as types.Context,
 			transformFacts,
@@ -1924,7 +1960,7 @@ export class DocumentIndex<
 				},
 				getIndexable: () => {
 					if (!hasCached) {
-						const transformed = this.transformer(
+						const transformed = this.transformWithDetachedFacts(
 							value,
 							projectionContext as types.Context,
 							transformFacts,
@@ -1961,7 +1997,11 @@ export class DocumentIndex<
 		) {
 			return;
 		}
-		const transformed = this.transformer(value, context, transformFacts);
+		const transformed = this.transformWithDetachedFacts(
+			value,
+			context,
+			transformFacts,
+		);
 		if (isPromiseLike(transformed)) {
 			return;
 		}
@@ -3150,7 +3190,11 @@ export class DocumentIndex<
 		this.cacheResolvedValue(idString, value);
 		const valueToIndex = this.transformerIsIdentity
 			? (value as any as I)
-			: await this.transformer(value, context, options?.transformFacts);
+			: await this.transformWithDetachedFacts(
+					value,
+					context,
+					options?.transformFacts,
+				);
 
 		coerceWithIndexed(value, valueToIndex);
 

package/src/transform.ts

@@ -43,6 +43,7 @@ export type DocumentTransformDescriptor =
 const NATIVE_DOCUMENT_TRANSFORM = Symbol.for(
 	"@peerbit/document/native-document-transform",
 );
+const builtInDocumentTransformers = new WeakSet<Function>();
 
 type DescribedDocumentTransformer<T, I> = DocumentTransformer<T, I> & {
 	readonly [NATIVE_DOCUMENT_TRANSFORM]?: DocumentTransformDescriptor;
@@ -142,9 +143,15 @@ const attachDocumentTransformDescriptor = <T, I>(
 		writable: false,
 		configurable: false,
 	});
+	builtInDocumentTransformers.add(fn);
 	return fn;
 };
 
+/** Distinguish library-created transforms from forgeable public descriptors. */
+export const isBuiltInDocumentTransformer = <T, I>(
+	transformer: DocumentTransformer<T, I> | undefined,
+): boolean => !!transformer && builtInDocumentTransformers.has(transformer);
+
 export const getDocumentTransformDescriptor = <T, I>(
 	transformer: DocumentTransformer<T, I> | undefined,
 ): DocumentTransformDescriptor | undefined =>