@peerbit/document 7.0.14 → 7.1.0-57b8640

package/src/program.ts

@@ -19,6 +19,7 @@ import {
 import { logger as loggerFn } from "@peerbit/logger";
 import { Program, type ProgramEvents } from "@peerbit/program";
 import {
+	type ReplicationDomain,
 	type SharedAppendOptions,
 	SharedLog,
 	type SharedLogOptions,
@@ -26,17 +27,19 @@ import {
 import { MAX_BATCH_SIZE } from "./constants.js";
 import {
 	BORSH_ENCODING_OPERATION,
-	type CanRead,
-	type CanSearch,
 	DeleteOperation,
-	DocumentIndex,
-	Operation,
+	type Operation,
 	PutOperation,
 	PutWithKeyOperation,
-	type TransformOptions,
 	coerceDeleteOperation,
 	isDeleteOperation,
 	isPutOperation,
+} from "./operation.js";
+import {
+	type CanRead,
+	type CanSearch,
+	DocumentIndex,
+	type TransformOptions,
 } from "./search.js";
 
 const logger = loggerFn({ module: "document" });
@@ -74,7 +77,11 @@ export type CanPerform<T> = (
 	properties: CanPerformOperations<T>,
 ) => MaybePromise<boolean>;
 
-export type SetupOptions<T, I = T> = {
+export type SetupOptions<
+	T,
+	I = T,
+	D extends ReplicationDomain<any, Operation> = any,
+> = {
 	type: AbstractType<T>;
 	canOpen?: (program: T) => MaybePromise<boolean>;
 	canPerform?: CanPerform<T>;
@@ -88,26 +95,29 @@ export type SetupOptions<T, I = T> = {
 		trim?: TrimOptions;
 	};
 	compatibility?: 6;
-} & Exclude<SharedLogOptions<Operation>, "compatibility">;
+} & Exclude<SharedLogOptions<Operation, D>, "compatibility">;
+
+export type ExtractArgs<T> =
+	T extends ReplicationDomain<infer Args, any> ? Args : never;
 
 @variant("documents")
 export class Documents<
 	T,
 	I extends Record<string, any> = T extends Record<string, any> ? T : any,
-> extends Program<SetupOptions<T, I>, DocumentEvents<T> & ProgramEvents> {
+	D extends ReplicationDomain<any, Operation> = any,
+> extends Program<SetupOptions<T, I, D>, DocumentEvents<T> & ProgramEvents> {
 	@field({ type: SharedLog })
-	log: SharedLog<Operation>;
+	log: SharedLog<Operation, D>;
 
 	@field({ type: "bool" })
 	immutable: boolean; // "Can I overwrite a document?"
 
 	@field({ type: DocumentIndex })
-	private _index: DocumentIndex<T, I>;
+	private _index: DocumentIndex<T, I, D>;
 
 	private _clazz!: AbstractType<T>;
 
 	private _optionCanPerform?: CanPerform<T>;
-	private _manuallySynced!: Set<string>;
 	private idResolver!: (any: any) => indexerTypes.IdPrimitive;
 
 	canOpen?: (program: T, entry: Entry<Operation>) => Promise<boolean> | boolean;
@@ -117,7 +127,7 @@ export class Documents<
 	constructor(properties?: {
 		id?: Uint8Array;
 		immutable?: boolean;
-		index?: DocumentIndex<T, I>;
+		index?: DocumentIndex<T, I, ExtractArgs<D>>;
 	}) {
 		super();
 
@@ -126,11 +136,11 @@ export class Documents<
 		this._index = properties?.index || new DocumentIndex();
 	}
 
-	get index(): DocumentIndex<T, I> {
+	get index(): DocumentIndex<T, I, D> {
 		return this._index;
 	}
 
-	async open(options: SetupOptions<T, I>) {
+	async open(options: SetupOptions<T, I, D>) {
 		this._clazz = options.type;
 		this.canOpen = options.canOpen;
 
@@ -144,7 +154,6 @@ export class Documents<
 		}
 
 		this._optionCanPerform = options.canPerform;
-		this._manuallySynced = new Set();
 		const idProperty = options.index?.idProperty || "id";
 		const idResolver =
 			options.id ||
@@ -167,12 +176,10 @@ export class Documents<
 				// here we arrive for all the results we want to persist.
 				// we we need to do here is
 				// 1. add the entry to a list of entries that we should persist through prunes
-				let heads: string[] = [];
-				for (const entry of result.results) {
-					this._manuallySynced.add(entry.context.gid);
-					heads.push(entry.context.head);
-				}
-				return this.log.log.join(heads);
+				await this.log.join(
+					result.results.map((x) => x.context.head),
+					{ replicate: true },
+				);
 			},
 			dbType: this.constructor,
 		});
@@ -185,11 +192,7 @@ export class Documents<
 			trim: options?.log?.trim,
 			replicate: options?.replicate,
 			replicas: options?.replicas,
-			sync: (entry: any) => {
-				// here we arrive when ever a insertion/pruning behaviour processes an entry
-				// returning true means that it should persist
-				return this._manuallySynced.has(entry.gid);
-			},
+			domain: options?.domain,
 
 			// document v6 and below need log compatibility of v8 or below
 			compatibility:
@@ -403,7 +406,10 @@ export class Documents<
 
 	public async put(
 		doc: T,
-		options?: SharedAppendOptions<Operation> & { unique?: boolean },
+		options?: SharedAppendOptions<Operation> & {
+			unique?: boolean;
+			replicate?: boolean;
+		},
 	) {
 		const keyValue = this.idResolver(doc);
 
@@ -424,7 +430,7 @@ export class Documents<
 			: (
 					await this._index.getDetailed(keyValue, {
 						local: true,
-						remote: { sync: true }, // only query remote if we know they exist
+						remote: { replicate: options?.replicate }, // only query remote if we know they exist
 					})
 				)?.[0]?.results[0];
 
@@ -458,6 +464,7 @@ export class Documents<
 			onChange: (change) => {
 				return this.handleChanges(change, { document: doc, operation });
 			},
+			replicate: options?.replicate,
 		});
 
 		return appended;
@@ -471,7 +478,7 @@ export class Documents<
 		const existing = (
 			await this._index.getDetailed(key, {
 				local: true,
-				remote: { sync: true },
+				remote: { replicate: options?.replicate },
 			})
 		)?.[0]?.results[0];
 
@@ -479,6 +486,7 @@ export class Documents<
 			throw new Error(`No entry with key '${key.primitive}' in the database`);
 		}
 
+		const entry = await this._resolveEntry(existing.context.head);
 		return this.log.append(
 			new DeleteOperation({
 				key,
@@ -486,7 +494,7 @@ export class Documents<
 			{
 				...options,
 				meta: {
-					next: [await this._resolveEntry(existing.context.head)],
+					next: [entry],
 					type: EntryType.CUT,
 					...options?.meta,
 				},
@@ -507,7 +515,7 @@ export class Documents<
 		}
 
 		const sortedEntries = [
-			...change.added,
+			...change.added.map((x) => x.entry),
 			...((await Promise.all(
 				change.removed.map((x) =>
 					x instanceof Entry ? x : this.log.log.entryIndex.get(x.hash),
@@ -545,7 +553,6 @@ export class Documents<
 
 					// get index key from value
 					const keyObject = this.idResolver(value);
-
 					const key = indexerTypes.toId(keyObject);
 
 					// document is already updated with more recent entry
@@ -574,8 +581,6 @@ export class Documents<
 					isPutOperation(payload) ||
 					removedSet.has(item.hash)
 				) {
-					this._manuallySynced.delete(item.meta.gid);
-
 					let value: T;
 					let key: indexerTypes.IdKey;
 

package/src/search.ts

@@ -18,12 +18,14 @@ import {
 	type RPCResponse,
 	queryAll,
 } from "@peerbit/rpc";
-import { SharedLog } from "@peerbit/shared-log";
+import { type ReplicationDomain, SharedLog } from "@peerbit/shared-log";
 import { SilentDelivery } from "@peerbit/stream-interface";
 import { AbortError } from "@peerbit/time";
 import { concat, fromString } from "uint8arrays";
 import { copySerialization } from "./borsh.js";
 import { MAX_BATCH_SIZE } from "./constants.js";
+import { type Operation, isPutOperation } from "./operation.js";
+import type { ExtractArgs } from "./program.js";
 
 const logger = loggerFn({ module: "document-index" });
 
@@ -34,119 +36,18 @@ type BufferedResult<T> = {
 	from: PublicSignKey;
 };
 
-@variant(0)
-export class Operation /* <T> */ {}
-
-export const BORSH_ENCODING_OPERATION = BORSH_ENCODING(Operation);
-
-// @deprecated
-@variant(0)
-export class PutWithKeyOperation extends Operation {
-	@field({ type: "string" })
-	key: string;
-
-	@field({ type: Uint8Array })
-	data: Uint8Array;
-
-	constructor(props: { key: string; data: Uint8Array }) {
-		super();
-		this.key = props.key;
-		this.data = props.data;
-	}
-}
-
-// @deprecated
-/* @variant(1)
-export class PutAllOperation<T> extends Operation<T> {
-	@field({ type: vec(PutOperation) })
-	docs: PutOperation<T>[];
-
-	constructor(props?: { docs: PutOperation<T>[] }) {
-		super();
-		if (props) {
-			this.docs = props.docs;
-		}
-	}
-}
- */
-
-// @deprecated
-@variant(2)
-export class DeleteByStringKeyOperation extends Operation {
-	@field({ type: "string" })
-	key: string;
-
-	constructor(props: { key: string }) {
-		super();
-		this.key = props.key;
-	}
-
-	toDeleteOperation(): DeleteOperation {
-		return new DeleteOperation({ key: indexerTypes.toId(this.key) });
-	}
-}
-
-export const coerceDeleteOperation = (
-	operation: DeleteOperation | DeleteByStringKeyOperation,
-): DeleteOperation => {
-	return operation instanceof DeleteByStringKeyOperation
-		? operation.toDeleteOperation()
-		: operation;
-};
-
-@variant(3)
-export class PutOperation extends Operation {
-	@field({ type: Uint8Array })
-	data: Uint8Array;
-
-	constructor(props: { data: Uint8Array }) {
-		super();
-		this.data = props.data;
-	}
-}
-
-export const isPutOperation = (
-	operation: Operation,
-): operation is PutOperation | PutWithKeyOperation => {
-	return (
-		operation instanceof PutOperation ||
-		operation instanceof PutWithKeyOperation
-	);
-};
-
-/**
- * Delete a document at a key
- */
-@variant(4)
-export class DeleteOperation extends Operation {
-	@field({ type: indexerTypes.IdKey })
-	key: indexerTypes.IdKey;
-
-	constructor(props: { key: indexerTypes.IdKey }) {
-		super();
-		this.key = props.key;
-	}
-}
-
-export const isDeleteOperation = (
-	operation: Operation,
-): operation is DeleteOperation | DeleteByStringKeyOperation => {
-	return (
-		operation instanceof DeleteOperation ||
-		operation instanceof DeleteByStringKeyOperation
-	);
-};
-
-export type RemoteQueryOptions<R> = RPCRequestAllOptions<R> & {
-	sync?: boolean;
+export type RemoteQueryOptions<R, D> = RPCRequestAllOptions<R> & {
+	replicate?: boolean;
 	minAge?: number;
 	throwOnMissing?: boolean;
+	domain?: ExtractArgs<D>;
+	eager?: boolean; // whether to query newly joined peers before they have matured
 };
-export type QueryOptions<R> = {
-	remote?: boolean | RemoteQueryOptions<types.AbstractSearchResult<R>>;
+export type QueryOptions<R, D> = {
+	remote?: boolean | RemoteQueryOptions<types.AbstractSearchResult<R>, D>;
 	local?: boolean;
 };
-export type SearchOptions<R> = QueryOptions<R>;
+export type SearchOptions<R, D> = QueryOptions<R, D>;
 
 type Transformer<T, I> = (obj: T, context: types.Context) => MaybePromise<I>;
 
@@ -156,18 +57,18 @@ export type ResultsIterator<T> = {
 	done: () => boolean;
 };
 
-type QueryDetailedOptions<T> = QueryOptions<T> & {
+type QueryDetailedOptions<T, D> = QueryOptions<T, D> & {
 	onResponse?: (
 		response: types.AbstractSearchResult<T>,
 		from: PublicSignKey,
 	) => void | Promise<void>;
 };
 
-const introduceEntries = async <T>(
+const introduceEntries = async <T, D>(
 	responses: RPCResponse<types.AbstractSearchResult<T>>[],
 	type: AbstractType<T>,
 	sync: (result: types.Results<T>) => Promise<void>,
-	options?: QueryDetailedOptions<T>,
+	options?: QueryDetailedOptions<T, D>,
 ): Promise<RPCResponse<types.Results<T>>[]> => {
 	const results: RPCResponse<types.Results<T>>[] = [];
 	for (const response of responses) {
@@ -177,7 +78,7 @@ const introduceEntries = async <T>(
 
 		if (response.response instanceof types.Results) {
 			response.response.results.forEach((r) => r.init(type));
-			if (typeof options?.remote !== "boolean" && options?.remote?.sync) {
+			if (typeof options?.remote !== "boolean" && options?.remote?.replicate) {
 				await sync(response.response);
 			}
 			options?.onResponse &&
@@ -290,11 +191,10 @@ const isTransformerWithFunction = <T, I>(
 	return (options as TransformerAsFunction<T, I>).transform != null;
 };
 
-export type OpenOptions<T, I> = {
+export type OpenOptions<T, I, D extends ReplicationDomain<any, Operation>> = {
 	documentType: AbstractType<T>;
-
 	dbType: AbstractType<types.IDocumentStore<T>>;
-	log: SharedLog<Operation>;
+	log: SharedLog<Operation, D>;
 	canRead?: CanRead<T>;
 	canSearch?: CanSearch;
 	sync: (result: types.Results<T>) => Promise<void>;
@@ -308,9 +208,11 @@ type IndexableClass<I> = new (
 ) => IDocumentWithContext<I>;
 
 @variant("documents_index")
-export class DocumentIndex<T, I extends Record<string, any>> extends Program<
-	OpenOptions<T, I>
-> {
+export class DocumentIndex<
+	T,
+	I extends Record<string, any>,
+	D extends ReplicationDomain<any, Operation>,
+> extends Program<OpenOptions<T, I, D>> {
 	@field({ type: RPC })
 	_query: RPC<
 		indexerTypes.AbstractSearchRequest,
@@ -342,7 +244,7 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 
 	private _sync: (result: types.Results<T>) => Promise<void>;
 
-	private _log: SharedLog<Operation>;
+	private _log: SharedLog<Operation, D>;
 
 	private _resolverProgramCache?: Map<string | number | bigint, T>;
 	private _resolverCache: Cache<T>;
@@ -372,7 +274,7 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 		return this._valueEncoding;
 	}
 
-	async open(properties: OpenOptions<T, I>) {
+	async open(properties: OpenOptions<T, I, D>) {
 		this._log = properties.log;
 
 		this.documentType = properties.documentType;
@@ -527,7 +429,7 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 
 	public async get(
 		key: indexerTypes.Ideable | indexerTypes.IdKey,
-		options?: QueryOptions<T>,
+		options?: QueryOptions<T, D>,
 	): Promise<T | undefined> {
 		return (
 			await this.getDetailed(
@@ -579,7 +481,7 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 
 	public async getDetailed(
 		key: indexerTypes.IdKey | indexerTypes.IdPrimitive,
-		options?: QueryOptions<T>,
+		options?: QueryOptions<T, D>,
 	): Promise<types.Results<T>[] | undefined> {
 		let results: types.Results<T>[] | undefined;
 		if (key instanceof Uint8Array) {
@@ -808,11 +710,12 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 	 */
 	public async queryDetailed(
 		queryRequest: indexerTypes.SearchRequest,
-		options?: QueryDetailedOptions<T>,
+		options?: QueryDetailedOptions<T, D>,
 	): Promise<types.Results<T>[]> {
 		const local = typeof options?.local === "boolean" ? options?.local : true;
-		let remote: RemoteQueryOptions<types.AbstractSearchResult<T>> | undefined =
-			undefined;
+		let remote:
+			| RemoteQueryOptions<types.AbstractSearchResult<T>, D>
+			| undefined = undefined;
 		if (typeof options?.remote === "boolean") {
 			if (options?.remote) {
 				remote = {};
@@ -829,7 +732,6 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 			remote.priority = 1;
 		}
 
-		const promises: Promise<types.Results<T>[] | undefined>[] = [];
 		if (!local && !remote) {
 			throw new Error(
 				"Expecting either 'options.remote' or 'options.local' to be true",
@@ -850,53 +752,53 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 			}
 		}
 
+		let resolved: types.Results<T>[] = [];
 		if (remote) {
-			const replicatorGroups = await this._log.getReplicatorUnion(
-				remote.minAge,
+			const replicatorGroups = await this._log.getCover(
+				remote.domain ?? (undefined as any),
+				{
+					roleAge: remote.minAge,
+					eager: remote.eager,
+				},
 			);
 
 			if (replicatorGroups) {
 				const groupHashes: string[][] = replicatorGroups.map((x) => [x]);
-				const fn = async () => {
-					const rs: types.Results<T>[] = [];
-					const responseHandler = async (
-						results: RPCResponse<types.AbstractSearchResult<T>>[],
-					) => {
-						for (const r of await introduceEntries(
-							results,
-							this.documentType,
-							this._sync,
-							options,
-						)) {
-							rs.push(r.response);
-						}
-					};
-					try {
-						if (queryRequest instanceof indexerTypes.CloseIteratorRequest) {
-							// don't wait for responses
-							await this._query.request(queryRequest, { mode: remote!.mode });
-						} else {
-							await queryAll(
-								this._query,
-								groupHashes,
-								queryRequest,
-								responseHandler,
-								remote,
-							);
-						}
-					} catch (error) {
-						if (error instanceof MissingResponsesError) {
-							logger.warn("Did not reciveve responses from all shard");
-							if (remote?.throwOnMissing) {
-								throw error;
-							}
-						} else {
+				const responseHandler = async (
+					results: RPCResponse<types.AbstractSearchResult<T>>[],
+				) => {
+					for (const r of await introduceEntries(
+						results,
+						this.documentType,
+						this._sync,
+						options,
+					)) {
+						resolved.push(r.response);
+					}
+				};
+				try {
+					if (queryRequest instanceof indexerTypes.CloseIteratorRequest) {
+						// don't wait for responses
+						await this._query.request(queryRequest, { mode: remote!.mode });
+					} else {
+						await queryAll(
+							this._query,
+							groupHashes,
+							queryRequest,
+							responseHandler,
+							remote,
+						);
+					}
+				} catch (error) {
+					if (error instanceof MissingResponsesError) {
+						logger.warn("Did not reciveve responses from all shard");
+						if (remote?.throwOnMissing) {
 							throw error;
 						}
+					} else {
+						throw error;
 					}
-					return rs;
-				};
-				promises.push(fn());
+				}
 			} else {
 				// TODO send without direction out to the world? or just assume we can insert?
 				/* 	promises.push(
@@ -909,7 +811,6 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 				); */
 			}
 		}
-		const resolved = await Promise.all(promises);
 		for (const r of resolved) {
 			if (r) {
 				if (r instanceof Array) {
@@ -930,7 +831,7 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 	 */
 	public async search(
 		queryRequest: indexerTypes.SearchRequest,
-		options?: SearchOptions<T>,
+		options?: SearchOptions<T, D>,
 	): Promise<T[]> {
 		// Set fetch to search size, or max value (default to max u32 (4294967295))
 		queryRequest.fetch = queryRequest.fetch ?? 0xffffffff;
@@ -966,7 +867,7 @@ export class DocumentIndex<T, I extends Record<string, any>> extends Program<
 	 */
 	public iterate(
 		queryRequest: indexerTypes.SearchRequest,
-		options?: QueryOptions<T>,
+		options?: QueryOptions<T, D>,
 	): ResultsIterator<T> {
 		let fetchPromise: Promise<any> | undefined = undefined;
 		const peerBufferMap: Map<