@peerbit/document 6.0.7-efee9d3 → 7.0.0-3a75d6e

package/src/program.ts

@@ -3,36 +3,37 @@ import {
 	BorshError,
 	field,
 	serialize,
-	variant
+	variant,
 } from "@dao-xyz/borsh";
-import { type Change, Entry, EntryType, type TrimOptions } from "@peerbit/log";
-import { Program, type ProgramEvents } from "@peerbit/program";
+import { CustomEvent } from "@libp2p/interface";
 import { AccessError, DecryptedThing } from "@peerbit/crypto";
+import * as documentsTypes from "@peerbit/document-interface";
+import * as indexerTypes from "@peerbit/indexer-interface";
+import {
+	type Change,
+	Entry,
+	EntryType,
+	type ShallowOrFullEntry,
+	type TrimOptions,
+} from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
-import { CustomEvent } from "@libp2p/interface";
+import { Program, type ProgramEvents } from "@peerbit/program";
 import {
-	type RoleOptions,
-	Observer,
-	Replicator,
+	type SharedAppendOptions,
 	SharedLog,
 	type SharedLogOptions,
-	type SharedAppendOptions
 } from "@peerbit/shared-log";
-import * as types from "@peerbit/document-interface";
-
-export type { RoleOptions }; // For convenience (so that consumers does not have to do the import above from shared-log packages)
-
+import { MAX_BATCH_SIZE } from "./constants.js";
 import {
-	type IndexableFields,
 	BORSH_ENCODING_OPERATION,
+	type CanRead,
+	type CanSearch,
 	DeleteOperation,
 	DocumentIndex,
 	Operation,
 	PutOperation,
-	type CanSearch,
-	type CanRead
+	type TransformOptions,
 } from "./search.js";
-import { MAX_BATCH_SIZE } from "./constants.js";
 
 const logger = loggerFn({ module: "document" });
 
@@ -66,30 +67,29 @@ type CanPerformDelete<T> = {
 
 export type CanPerformOperations<T> = CanPerformPut<T> | CanPerformDelete<T>;
 export type CanPerform<T> = (
-	properties: CanPerformOperations<T>
+	properties: CanPerformOperations<T>,
 ) => MaybePromise<boolean>;
 
-export type SetupOptions<T> = {
+export type SetupOptions<T, I = T> = {
 	type: AbstractType<T>;
 	canOpen?: (program: T) => MaybePromise<boolean>;
 	canPerform?: CanPerform<T>;
-	id?: (obj: any) => types.IdPrimitive;
+	id?: (obj: any) => indexerTypes.IdPrimitive;
 	index?: {
-		idProperty?: string | string[];
 		canSearch?: CanSearch;
 		canRead?: CanRead<T>;
-		fields?: IndexableFields<T>;
-	};
+		idProperty?: string | string[];
+	} & TransformOptions<T, I>;
 	log?: {
 		trim?: TrimOptions;
 	};
 } & SharedLogOptions<Operation>;
 
 @variant("documents")
-export class Documents<T> extends Program<
-	SetupOptions<T>,
-	DocumentEvents<T> & ProgramEvents
-> {
+export class Documents<
+	T,
+	I extends Record<string, any> = T extends Record<string, any> ? T : any,
+> extends Program<SetupOptions<T, I>, DocumentEvents<T> & ProgramEvents> {
 	@field({ type: SharedLog })
 	log: SharedLog<Operation>;
 
@@ -97,20 +97,20 @@ export class Documents<T> extends Program<
 	immutable: boolean; // "Can I overwrite a document?"
 
 	@field({ type: DocumentIndex })
-	private _index: DocumentIndex<T>;
+	private _index: DocumentIndex<T, I>;
 
-	private _clazz: AbstractType<T>;
+	private _clazz!: AbstractType<T>;
 
 	private _optionCanPerform?: CanPerform<T>;
-	private _manuallySynced: Set<string>;
-	private idResolver: (any: any) => types.IdPrimitive;
+	private _manuallySynced!: Set<string>;
+	private idResolver!: (any: any) => indexerTypes.IdPrimitive;
 
 	canOpen?: (program: T, entry: Entry<Operation>) => Promise<boolean> | boolean;
 
 	constructor(properties?: {
 		id?: Uint8Array;
 		immutable?: boolean;
-		index?: DocumentIndex<T>;
+		index?: DocumentIndex<T, I>;
 	}) {
 		super();
 
@@ -119,11 +119,11 @@ export class Documents<T> extends Program<
 		this._index = properties?.index || new DocumentIndex();
 	}
 
-	get index(): DocumentIndex<T> {
+	get index(): DocumentIndex<T, I> {
 		return this._index;
 	}
 
-	async open(options: SetupOptions<T>) {
+	async open(options: SetupOptions<T, I>) {
 		this._clazz = options.type;
 		this.canOpen = options.canOpen;
 
@@ -131,7 +131,7 @@ export class Documents<T> extends Program<
 		if (Program.isPrototypeOf(this._clazz)) {
 			if (!this.canOpen) {
 				throw new Error(
-					"Document store needs to be opened with canOpen option when the document type is a Program"
+					"Document store needs to be opened with canOpen option when the document type is a Program",
 				);
 			}
 		}
@@ -143,28 +143,19 @@ export class Documents<T> extends Program<
 			options.id ||
 			(typeof idProperty === "string"
 				? (obj: any) => obj[idProperty as string]
-				: (obj: any) => types.extractFieldValue(obj, idProperty as string[]));
+				: (obj: any) =>
+						indexerTypes.extractFieldValue(obj, idProperty as string[]));
 
 		this.idResolver = idResolver;
 
-		let transform: IndexableFields<T>;
-		if (options.index?.fields) {
-			if (typeof options.index.fields === "function") {
-				transform = options.index.fields;
-			} else {
-				transform = options.index.fields;
-			}
-		} else {
-			transform = (obj) => obj as Record<string, any>; // TODO check types
-		}
 		await this._index.open({
-			type: this._clazz,
 			log: this.log,
 			canRead: options?.index?.canRead,
 			canSearch: options.index?.canSearch,
-			fields: transform,
+			documentType: this._clazz,
+			transform: options.index,
 			indexBy: idProperty,
-			sync: async (result: types.Results<T>) => {
+			sync: async (result: documentsTypes.Results<T>) => {
 				// here we arrive for all the results we want to persist.
 				// we we need to do here is
 				// 1. add the entry to a list of entries that we should persist through prunes
@@ -175,7 +166,7 @@ export class Documents<T> extends Program<
 				}
 				return this.log.log.join(heads);
 			},
-			dbType: this.constructor
+			dbType: this.constructor,
 		});
 
 		await this.log.open({
@@ -184,13 +175,13 @@ export class Documents<T> extends Program<
 			canAppend: this.canAppend.bind(this),
 			onChange: this.handleChanges.bind(this),
 			trim: options?.log?.trim,
-			role: options?.role,
+			replicate: options?.replicate,
 			replicas: options?.replicas,
 			sync: (entry: any) => {
 				// here we arrive when ever a insertion/pruning behaviour processes an entry
 				// returning true means that it should persist
 				return this._manuallySynced.has(entry.gid);
-			}
+			},
 		});
 	}
 
@@ -201,20 +192,13 @@ export class Documents<T> extends Program<
 	private async _resolveEntry(history: Entry<Operation> | string) {
 		return typeof history === "string"
 			? (await this.log.log.get(history)) ||
-			(await Entry.fromMultihash<Operation>(this.log.log.blocks, history))
+					(await Entry.fromMultihash<Operation>(this.log.log.blocks, history))
 			: history;
 	}
 
-	async updateRole(role: RoleOptions) {
-		await this.log.updateRole(role);
-	}
-	get role(): Replicator | Observer {
-		return this.log.role;
-	}
-
 	async canAppend(
 		entry: Entry<Operation>,
-		reference?: { document: T; operation: PutOperation }
+		reference?: { document: T; operation: PutOperation },
 	): Promise<boolean> {
 		const l0 = await this._canAppend(entry as Entry<Operation>, reference);
 		if (!l0) {
@@ -244,16 +228,16 @@ export class Documents<T> extends Program<
 					!(await this._optionCanPerform(
 						operation instanceof PutOperation
 							? {
-								type: "put",
-								value: document!,
-								operation,
-								entry: entry as any as Entry<PutOperation>
-							}
+									type: "put",
+									value: document!,
+									operation,
+									entry: entry as any as Entry<PutOperation>,
+								}
 							: {
-								type: "delete",
-								operation,
-								entry: entry as any as Entry<DeleteOperation>
-							}
+									type: "delete",
+									operation,
+									entry: entry as any as Entry<DeleteOperation>,
+								},
 					))
 				) {
 					return false;
@@ -272,12 +256,12 @@ export class Documents<T> extends Program<
 
 	async _canAppend(
 		entry: Entry<Operation>,
-		reference?: { document: T; operation: PutOperation }
+		reference?: { document: T; operation: PutOperation },
 	): Promise<PutOperation | DeleteOperation | false> {
 		const resolve = async (history: Entry<Operation> | string) => {
 			return typeof history === "string"
 				? this.log.log.get(history) ||
-				(await Entry.fromMultihash(this.log.log.blocks, history))
+						(await Entry.fromMultihash(this.log.log.blocks, history))
 				: history;
 		};
 		const pointsToHistory = async (history: Entry<Operation> | string) => {
@@ -301,7 +285,7 @@ export class Documents<T> extends Program<
 		try {
 			entry.init({
 				encoding: this.log.log.encoding,
-				keychain: this.node.services.keychain
+				keychain: this.node.services.keychain,
 			});
 			const operation =
 				reference?.operation || entry._payload instanceof DecryptedThing
@@ -315,9 +299,10 @@ export class Documents<T> extends Program<
 					this.index.valueEncoding.decoder(putOperation.data);
 				const keyValue = this.idResolver(value);
 
-				const key = types.toId(keyValue);
+				const key = indexerTypes.toId(keyValue);
 
-				const existingDocument = await this.index.engine.get(key);
+				const existingDocument = (await this.index.getDetailed(key))?.[0]
+					?.results[0];
 				if (existingDocument && existingDocument.context.head !== entry.hash) {
 					//  econd condition can false if we reset the operation log, while not  resetting the index. For example when doing .recover
 					if (this.immutable) {
@@ -344,7 +329,9 @@ export class Documents<T> extends Program<
 				if (entry.next.length !== 1) {
 					return false;
 				}
-				const existingDocument = await this._index.engine.get(operation.key);
+				const existingDocument = (
+					await this.index.getDetailed(operation.key)
+				)?.[0].results[0];
 				if (!existingDocument) {
 					// already deleted
 					return operation; // assume ok
@@ -377,32 +364,33 @@ export class Documents<T> extends Program<
 
 	public async put(
 		doc: T,
-		options?: SharedAppendOptions<Operation> & { unique?: boolean }
+		options?: SharedAppendOptions<Operation> & { unique?: boolean },
 	) {
 		const keyValue = this.idResolver(doc);
 
 		// type check the key
-		types.checkId(keyValue);
+		indexerTypes.checkId(keyValue);
 
 		const ser = serialize(doc);
 		if (ser.length > MAX_BATCH_SIZE) {
 			throw new Error(
-				`Document is too large (${ser.length * 1e-6
-				}) mb). Needs to be less than ${MAX_BATCH_SIZE * 1e-6} mb`
+				`Document is too large (${
+					ser.length * 1e-6
+				}) mb). Needs to be less than ${MAX_BATCH_SIZE * 1e-6} mb`,
 			);
 		}
 
 		const existingDocument = options?.unique
 			? undefined
 			: (
-				await this._index.getDetailed(keyValue, {
-					local: true,
-					remote: { sync: true } // only query remote if we know they exist
-				})
-			)?.[0]?.results[0];
+					await this._index.getDetailed(keyValue, {
+						local: true,
+						remote: { sync: true }, // only query remote if we know they exist
+					})
+				)?.[0]?.results[0];
 
 		const operation = new PutOperation({
-			data: ser
+			data: ser,
 		});
 		const appended = await this.log.append(operation, {
 			...options,
@@ -410,73 +398,86 @@ export class Documents<T> extends Program<
 				next: existingDocument
 					? [await this._resolveEntry(existingDocument.context.head)]
 					: [],
-				...options?.meta
+				...options?.meta,
 			},
 			canAppend: (entry) => {
 				return this.canAppend(entry, { document: doc, operation });
 			},
 			onChange: (change) => {
 				return this.handleChanges(change, { document: doc, operation });
-			}
+			},
 		});
 
 		return appended;
 	}
 
-	async del(id: types.Ideable, options?: SharedAppendOptions<Operation>) {
-		const key = types.toId(id);
+	async del(
+		id: indexerTypes.Ideable,
+		options?: SharedAppendOptions<Operation>,
+	) {
+		const key = indexerTypes.toId(id);
 		const existing = (
 			await this._index.getDetailed(key, {
 				local: true,
-				remote: { sync: true }
+				remote: { sync: true },
 			})
 		)?.[0]?.results[0];
 
 		if (!existing) {
-			throw new Error(`No entry with key '${key}' in the database`);
+			throw new Error(`No entry with key '${key.primitive}' in the database`);
 		}
 
 		return this.log.append(
 			new DeleteOperation({
-				key
+				key,
 			}),
 			{
 				...options,
 				meta: {
 					next: [await this._resolveEntry(existing.context.head)],
 					type: EntryType.CUT,
-					...options?.meta
-				}
-			} //
+					...options?.meta,
+				},
+			}, //
 		);
 	}
 
 	async handleChanges(
 		change: Change<Operation>,
-		reference?: { document: T; operation: PutOperation }
+		reference?: { document: T; operation: PutOperation },
 	): Promise<void> {
 		const isAppendOperation =
 			change?.added.length === 1 ? !!change.added[0] : false;
 
-		const removed = [...(change.removed || [])];
-		const removedSet = new Map<string, Entry<Operation>>();
-		for (const r of removed) {
+		const removedSet = new Map<string, ShallowOrFullEntry<Operation>>();
+		for (const r of change.removed) {
 			removedSet.set(r.hash, r);
 		}
-		const sortedEntries = [...change.added, ...(removed || [])]
-			.sort(this.log.log.sortFn)
-			.reverse(); // sort so we get newest to oldest
+		const sortedEntries = [
+			...change.added,
+			...((await Promise.all(
+				change.removed.map((x) =>
+					x instanceof Entry ? x : this.log.log.entryIndex.get(x.hash),
+				),
+			)) || []),
+		]; // TODO assert sorting
+		/* 
+				const sortedEntries = [...change.added, ...(removed || [])]
+					.sort(this.log.log.sortFn)
+					.reverse(); // sort so we get newest to oldest */
 
 		// There might be a case where change.added and change.removed contains the same document id. Usaully because you use the "trim" option
 		// in combination with inserting the same document. To mitigate this, we loop through the changes and modify the behaviour for this
 
 		let documentsChanged: DocumentsChange<T> = {
 			added: [],
-			removed: []
+			removed: [],
 		};
 
 		let modified: Set<string | number | bigint> = new Set();
 		for (const item of sortedEntries) {
+			if (!item) continue;
+
 			try {
 				const payload =
 					item._payload instanceof DecryptedThing
@@ -493,7 +494,7 @@ export class Documents<T> extends Program<
 					// get index key from value
 					const keyObject = this.idResolver(value);
 
-					const key = types.toId(keyObject);
+					const key = indexerTypes.toId(keyObject);
 
 					// document is already updated with more recent entry
 					if (modified.has(key.primitive)) {
@@ -505,17 +506,16 @@ export class Documents<T> extends Program<
 						// if replicator, then open
 						if (
 							(await this.canOpen!(value, item)) &&
-							this.log.role instanceof Replicator &&
-							(await this.log.replicator(item)) // TODO types, throw runtime error if replicator is not provided
+							(await this.log.isReplicator(item)) // TODO types, throw runtime error if replicator is not provided
 						) {
 							value = (await this.node.open(value, {
 								parent: this as Program<any, any>,
-								existing: "reuse"
+								existing: "reuse",
 							})) as any as T; // TODO types
 						}
 					}
 					documentsChanged.added.push(value);
-					this._index.put(value, item, key);
+					await this._index.put(value, item, key);
 					modified.add(key.primitive);
 				} else if (
 					(payload instanceof DeleteOperation && !removedSet.has(item.hash)) ||
@@ -525,24 +525,24 @@ export class Documents<T> extends Program<
 					this._manuallySynced.delete(item.gid);
 
 					let value: T;
-					let key: string | number | bigint;
+					let key: indexerTypes.IdKey;
 
 					if (payload instanceof PutOperation) {
 						value = this.index.valueEncoding.decoder(payload.data);
-						key = types.toIdeable(this.idResolver(value));
+						key = indexerTypes.toId(this.idResolver(value));
 						// document is already updated with more recent entry
-						if (modified.has(key)) {
+						if (modified.has(key.primitive)) {
 							continue;
 						}
 					} else if (payload instanceof DeleteOperation) {
-						key = payload.key.primitive;
+						key = payload.key;
 						// document is already updated with more recent entry
-						if (modified.has(key)) {
+						if (modified.has(key.primitive)) {
 							continue;
 						}
 						const document = await this._index.get(key, {
 							local: true,
-							remote: false
+							remote: false,
 						});
 						if (!document) {
 							continue;
@@ -559,11 +559,11 @@ export class Documents<T> extends Program<
 					}
 
 					// update index
-					this._index.del(key);
-
-					modified.add(key);
+					await this._index.del(key);
+					modified.add(key.primitive);
 				} else {
 					// Unknown operation
+					throw new OperationError("Unknown operation");
 				}
 			} catch (error) {
 				if (error instanceof AccessError) {
@@ -574,7 +574,7 @@ export class Documents<T> extends Program<
 		}
 
 		this.events.dispatchEvent(
-			new CustomEvent("change", { detail: documentsChanged })
+			new CustomEvent("change", { detail: documentsChanged }),
 		);
 	}
 }