@peerbit/document 8.0.6 → 8.1.0-43685b6

package/src/domain.ts

@@ -1,56 +1,146 @@
+import type * as types from "@peerbit/document-interface";
 import { Entry, type ShallowEntry } from "@peerbit/log";
+import { logger as loggerFn } from "@peerbit/logger";
 import {
 	type EntryReplicated,
+	MAX_U32,
+	MAX_U64,
+	type NumberFromType,
 	type ReplicationDomain,
+	type SharedLog,
 } from "@peerbit/shared-log";
-import {
-	type Documents,
-	type Operation,
-	isPutOperation,
-} from "../src/index.js";
-
-type RangeArgs = { from: number; to: number };
-export type CustomDomain = ReplicationDomain<RangeArgs, Operation, "u32">;
-
-export const createDocumentDomain = <T extends object>(
-	db: Documents<T, any, CustomDomain>,
-	options: {
-		fromValue: (value: T) => number;
-		fromMissing?: (
-			entry: EntryReplicated<"u32"> | ShallowEntry | Entry<Operation>,
-		) => number;
-	},
-): CustomDomain => {
-	const fromValue = options.fromValue;
-	const fromMissing = options.fromMissing || (() => 0xffffffff);
-	return {
-		type: "custom",
-		resolution: "u32",
-		fromArgs(args, log) {
-			if (!args) {
-				return { offset: log.node.identity.publicKey };
-			}
-			return {
-				offset: args.from,
-				length: args.to - args.from,
-			};
-		},
-		fromEntry: async (entry) => {
-			const item = await (
-				entry instanceof Entry ? entry : await db.log.log.get(entry.hash)
-			)?.getPayloadValue();
-			if (!item) {
-				// eslint-disable-next-line no-console
-				console.error("Item not found");
-				return fromMissing(entry);
-			}
-
-			if (isPutOperation(item)) {
-				const document = db.index.valueEncoding.decoder(item.data);
-				return fromValue(document);
-			}
-
-			return fromMissing(entry);
-		},
-	};
+import { type Operation, isPutOperation } from "./operation.js";
+import type { DocumentIndex } from "./search.js";
+
+const logger = loggerFn({ module: "document-domain" });
+
+type InferT<D> = D extends Documents<infer T, any, any> ? T : never;
+type InferR<D> =
+	D extends Documents<any, ReplicationDomain<any, any, infer R>> ? R : never;
+
+type Documents<
+	T,
+	D extends ReplicationDomain<any, Operation, R>,
+	R extends "u32" | "u64" = D extends ReplicationDomain<any, T, infer I>
+		? I
+		: "u32",
+> = { log: SharedLog<Operation, D, R>; index: DocumentIndex<T, any, D> };
+
+type RangeArgs<R extends "u32" | "u64"> = {
+	from: NumberFromType<R>;
+	to: NumberFromType<R>;
+};
+export type CustomDocumentDomain<R extends "u32" | "u64"> = ReplicationDomain<
+	RangeArgs<R>,
+	Operation,
+	R
+> & { canProjectToOneSegment: (request: types.SearchRequest) => boolean };
+
+type FromEntry<R extends "u32" | "u64"> = {
+	fromEntry?: (
+		entry: ShallowEntry | Entry<Operation> | EntryReplicated<R>,
+	) => NumberFromType<R>;
 };
+type FromValue<T, R extends "u32" | "u64"> = {
+	fromValue?: (
+		value: T | undefined,
+		entry: ShallowEntry | Entry<Operation> | EntryReplicated<R>,
+	) => NumberFromType<R>;
+};
+
+type CreateArgs<
+	R extends "u32" | "u64",
+	DB extends Documents<any, any, any>,
+> = {
+	resolution: R;
+	canProjectToOneSegment: (request: types.SearchRequest) => boolean;
+	mergeSegmentMaxDelta?: number;
+} & (FromEntry<R> | FromValue<InferT<DB>, R>);
+
+export const createDocumentDomainFromProperty = <
+	R extends "u32" | "u64",
+	DB extends Documents<any, any, any>,
+>(properties: {
+	property: keyof InferT<DB>;
+	resolution: R;
+	mergeSegmentMaxDelta?: number;
+}): ((db: DB) => CustomDocumentDomain<InferR<DB>>) => {
+	const coerceNumber = (number: number | bigint): NumberFromType<R> =>
+		(properties.resolution === "u32"
+			? number
+			: BigInt(number)) as NumberFromType<R>;
+	return createDocumentDomain({
+		resolution: properties.resolution,
+		canProjectToOneSegment: (request) =>
+			request.sort[0]?.key[0] === properties.property,
+		fromValue: (value) => coerceNumber(value![properties.property]),
+		mergeSegmentMaxDelta: properties.mergeSegmentMaxDelta,
+	});
+};
+
+export const createDocumentDomain =
+	<R extends "u32" | "u64", DB extends Documents<any, any, any>>(
+		args: CreateArgs<R, DB>,
+	): ((db: DB) => CustomDocumentDomain<InferR<DB>>) =>
+	(db: DB) => {
+		let maxValue = args.resolution === "u32" ? MAX_U32 : MAX_U64;
+		let fromEntry = (args as FromEntry<InferR<DB>>).fromEntry
+			? (args as FromEntry<InferR<DB>>).fromEntry!
+			: async (
+					entry: ShallowEntry | Entry<Operation> | EntryReplicated<any>,
+				) => {
+					const item = await (
+						entry instanceof Entry ? entry : await db.log.log.get(entry.hash)
+					)?.getPayloadValue();
+
+					let document: InferT<DB> | undefined = undefined;
+					if (!item) {
+						logger.error("Item not found");
+					} else if (isPutOperation(item)) {
+						document = db.index.valueEncoding.decoder(item.data);
+					}
+					return (args as FromValue<any, any>).fromValue!(
+						document,
+						entry,
+					) as NumberFromType<InferR<DB>>;
+				};
+		return {
+			type: "custom",
+			resolution: args.resolution as InferR<DB>,
+			canProjectToOneSegment: args.canProjectToOneSegment,
+			fromArgs(args) {
+				if (!args) {
+					return {
+						offset: db.log.node.identity.publicKey,
+						length: maxValue as NumberFromType<InferR<DB>>,
+					};
+				}
+				return {
+					offset: args.from,
+					length: (args.to - args.from) as NumberFromType<InferR<DB>>,
+				};
+			},
+			fromEntry,
+			canMerge:
+				args.mergeSegmentMaxDelta == null
+					? undefined
+					: (from, into) => {
+							if (
+								Math.abs(Number(from.end2 - into.start1)) <=
+								args.mergeSegmentMaxDelta!
+							) {
+								return true;
+							}
+							if (
+								Math.abs(Number(from.start1 - into.end2)) <=
+								args.mergeSegmentMaxDelta!
+							) {
+								return true;
+							}
+							if (from.overlaps(into)) {
+								return true;
+							}
+							return false;
+						},
+		};
+	};

package/src/index.ts

@@ -18,3 +18,8 @@ export type {
 export * from "./operation.js";
 export { MAX_BATCH_SIZE as MAX_DOCUMENT_SIZE } from "./constants.js";
 export { ClosedError } from "@peerbit/program";
+export {
+	type CustomDocumentDomain,
+	createDocumentDomain,
+	createDocumentDomainFromProperty,
+} from "./domain.js";

package/src/program.ts

@@ -24,6 +24,7 @@ import {
 	type SharedLogOptions,
 } from "@peerbit/shared-log";
 import { MAX_BATCH_SIZE } from "./constants.js";
+import type { CustomDocumentDomain } from "./domain.js";
 import {
 	BORSH_ENCODING_OPERATION,
 	DeleteOperation,
@@ -76,9 +77,11 @@ export type CanPerform<T> = (
 	properties: CanPerformOperations<T>,
 ) => MaybePromise<boolean>;
 
+type InferR<D> = D extends ReplicationDomain<any, any, infer I> ? I : "u32";
+
 export type SetupOptions<
 	T,
-	I = T,
+	I extends Record<string, any> = T extends Record<string, any> ? T : any,
 	D extends ReplicationDomain<any, Operation, any> = any,
 > = {
 	type: AbstractType<T>;
@@ -94,7 +97,8 @@ export type SetupOptions<
 		trim?: TrimOptions;
 	};
 	compatibility?: 6 | 7;
-} & Exclude<SharedLogOptions<Operation, D>, "compatibility">;
+	domain?: (db: Documents<T, I, D>) => CustomDocumentDomain<InferR<D>>;
+} & Omit<SharedLogOptions<Operation, D, InferR<D>>, "compatibility" | "domain">;
 
 export type ExtractArgs<T> =
 	T extends ReplicationDomain<infer Args, any, any> ? Args : never;
@@ -106,7 +110,7 @@ export class Documents<
 	D extends ReplicationDomain<any, Operation, any> = any,
 > extends Program<SetupOptions<T, I, D>, DocumentEvents<T> & ProgramEvents> {
 	@field({ type: SharedLog })
-	log: SharedLog<Operation, D>;
+	log: SharedLog<Operation, D, InferR<D>>;
 
 	@field({ type: "bool" })
 	immutable: boolean; // "Can I overwrite a document?"
@@ -118,6 +122,7 @@ export class Documents<
 
 	private _optionCanPerform?: CanPerform<T>;
 	private idResolver!: (any: any) => indexerTypes.IdPrimitive;
+	private domain?: CustomDocumentDomain<InferR<D>>;
 
 	canOpen?: (program: T, entry: Entry<Operation>) => Promise<boolean> | boolean;
 
@@ -171,13 +176,16 @@ export class Documents<
 			documentType: this._clazz,
 			transform: options.index,
 			indexBy: idProperty,
-			sync: async (result: documentsTypes.Results<T>) => {
+			sync: async (
+				query: documentsTypes.SearchRequest,
+				result: documentsTypes.Results<T>,
+			) => {
 				// here we arrive for all the results we want to persist.
-				// we we need to do here is
-				// 1. add the entry to a list of entries that we should persist through prunes
+
+				let mergeSegments = this.domain?.canProjectToOneSegment(query);
 				await this.log.join(
 					result.results.map((x) => x.context.head),
-					{ replicate: true },
+					{ replicate: { assumeSynced: true, mergeSegments } },
 				);
 			},
 			dbType: this.constructor,
@@ -191,6 +199,8 @@ export class Documents<
 		} else if (options.compatibility === 7) {
 			logCompatiblity = 9;
 		}
+
+		this.domain = options.domain?.(this);
 		await this.log.open({
 			encoding: BORSH_ENCODING_OPERATION,
 			canReplicate: options?.canReplicate,
@@ -199,7 +209,9 @@ export class Documents<
 			trim: options?.log?.trim,
 			replicate: options?.replicate,
 			replicas: options?.replicas,
-			domain: options?.domain,
+			domain: (options?.domain
+				? (log: any) => options.domain!(this)
+				: undefined) as any, /// TODO types,
 			compatibility: logCompatiblity,
 		});
 	}

package/src/resumable-iterator.ts

@@ -1,6 +1,5 @@
 import { Cache } from "@peerbit/cache";
 import {
-	type CloseIteratorRequest,
 	type CollectNextRequest,
 	SearchRequest,
 } from "@peerbit/document-interface";
@@ -44,7 +43,7 @@ export class ResumableIterators<T extends Record<string, any>> {
 		return next;
 	}
 
-	async close(close: CloseIteratorRequest) {
+	close(close: { idString: string }) {
 		this.clear(close.idString);
 	}
 

package/src/search.ts

@@ -56,6 +56,7 @@ export type ResultsIterator<T> = {
 	close: () => Promise<void>;
 	next: (number: number) => Promise<T[]>;
 	done: () => boolean;
+	all: () => Promise<T[]>;
 };
 
 type QueryDetailedOptions<T, D> = QueryOptions<T, D> & {
@@ -66,9 +67,13 @@ type QueryDetailedOptions<T, D> = QueryOptions<T, D> & {
 };
 
 const introduceEntries = async <T, D>(
+	queryRequest: types.SearchRequest,
 	responses: RPCResponse<types.AbstractSearchResult<T>>[],
 	type: AbstractType<T>,
-	sync: (result: types.Results<T>) => Promise<void>,
+	sync: (
+		queryRequest: types.SearchRequest,
+		result: types.Results<T>,
+	) => Promise<void>,
 	options?: QueryDetailedOptions<T, D>,
 ): Promise<RPCResponse<types.Results<T>>[]> => {
 	const results: RPCResponse<types.Results<T>>[] = [];
@@ -80,7 +85,7 @@ const introduceEntries = async <T, D>(
 		if (response.response instanceof types.Results) {
 			response.response.results.forEach((r) => r.init(type));
 			if (typeof options?.remote !== "boolean" && options?.remote?.replicate) {
-				await sync(response.response);
+				await sync(queryRequest, response.response);
 			}
 			options?.onResponse &&
 				(await options.onResponse(response.response, response.from!)); // TODO fix types
@@ -199,10 +204,13 @@ export type OpenOptions<
 > = {
 	documentType: AbstractType<T>;
 	dbType: AbstractType<types.IDocumentStore<T>>;
-	log: SharedLog<Operation, D>;
+	log: SharedLog<Operation, D, any>;
 	canRead?: CanRead<T>;
 	canSearch?: CanSearch;
-	sync: (result: types.Results<T>) => Promise<void>;
+	sync: (
+		request: types.SearchRequest,
+		result: types.Results<T>,
+	) => Promise<void>;
 	indexBy?: string | string[];
 	transform?: TransformOptions<T, I>;
 };
@@ -245,9 +253,12 @@ export class DocumentIndex<
 
 	private _valueEncoding: Encoding<T>;
 
-	private _sync: (result: types.Results<T>) => Promise<void>;
+	private _sync: (
+		reques: types.SearchRequest,
+		result: types.Results<T>,
+	) => Promise<void>;
 
-	private _log: SharedLog<Operation, D>;
+	private _log: SharedLog<Operation, D, any>;
 
 	private _resolverProgramCache?: Map<string | number | bigint, T>;
 	private _resolverCache: Cache<T>;
@@ -413,6 +424,7 @@ export class DocumentIndex<
 	async close(from?: Program): Promise<boolean> {
 		const closed = await super.close(from);
 		if (closed) {
+			this.clearAllResultQueues();
 			await this.index.stop?.();
 		}
 		return closed;
@@ -421,6 +433,7 @@ export class DocumentIndex<
 	async drop(from?: Program): Promise<boolean> {
 		const dropped = await super.drop(from);
 		if (dropped) {
+			this.clearAllResultQueues();
 			await this.index.drop?.();
 			await this.index.stop?.();
 		}
@@ -629,6 +642,7 @@ export class DocumentIndex<
 
 		if (prevQueued) {
 			this._resultQueue.delete(query.idString);
+			clearTimeout(prevQueued.timeout);
 			prevQueued = undefined;
 		}
 
@@ -684,7 +698,7 @@ export class DocumentIndex<
 		return results;
 	}
 
-	clearResultsQueue(
+	private clearResultsQueue(
 		query:
 			| types.SearchRequest
 			| types.CollectNextRequest
@@ -696,6 +710,15 @@ export class DocumentIndex<
 			this._resultQueue.delete(query.idString);
 		}
 	}
+
+	private clearAllResultQueues() {
+		for (const [key, queue] of this._resultQueue) {
+			clearTimeout(queue.timeout);
+			this._resultQueue.delete(key);
+			this._resumableIterators.close({ idString: key });
+		}
+	}
+
 	async processCloseIteratorRequest(
 		query: types.CloseIteratorRequest,
 		publicKey: PublicSignKey,
@@ -775,6 +798,7 @@ export class DocumentIndex<
 					results: RPCResponse<types.AbstractSearchResult<T>>[],
 				) => {
 					for (const r of await introduceEntries(
+						queryRequest,
 						results,
 						this.documentType,
 						this._sync,
@@ -1062,6 +1086,7 @@ export class DocumentIndex<
 								})
 								.then((response) =>
 									introduceEntries(
+										queryRequest,
 										response,
 										this.documentType,
 										this._sync,
@@ -1212,11 +1237,19 @@ export class DocumentIndex<
 			}
 			await Promise.all(promises);
 		};
-
+		let doneFn = () => done;
 		return {
 			close,
 			next,
-			done: () => done,
+			done: doneFn,
+			all: async () => {
+				let result: T[] = [];
+				while (doneFn() !== true) {
+					let batch = await next(100);
+					result.push(...batch);
+				}
+				return result;
+			},
 		};
 	}
 }