@peerbit/document 9.11.7 → 9.11.8-863fe8e

package/src/search.ts

@@ -1,5 +1,5 @@
 import { type AbstractType, field, serialize, variant } from "@dao-xyz/borsh";
-import type { PeerId } from "@libp2p/interface";
+import type { PeerId, TypedEventTarget } from "@libp2p/interface";
 import { Cache } from "@peerbit/cache";
 import {
 	type MaybePromise,
@@ -28,9 +28,11 @@ import {
 } from "@peerbit/shared-log";
 import { DataMessage, SilentDelivery } from "@peerbit/stream-interface";
 import { AbortError, waitFor } from "@peerbit/time";
+import pDefer from "p-defer";
 import { concat, fromString } from "uint8arrays";
 import { copySerialization } from "./borsh.js";
 import { MAX_BATCH_SIZE } from "./constants.js";
+import type { DocumentEvents, DocumentsChange } from "./events.js";
 import type { QueryPredictor } from "./most-common-query-predictor.js";
 import MostCommonQueryPredictor from "./most-common-query-predictor.js";
 import { type Operation, isPutOperation } from "./operation.js";
@@ -76,6 +78,21 @@ export type QueryOptions<R, D, Resolve extends boolean | undefined> = {
 	resolve?: Resolve;
 	signal?: AbortSignal;
 };
+
+export type GetOptions<R, D, Resolve extends boolean | undefined> = {
+	remote?:
+		| boolean
+		| RemoteQueryOptions<
+				types.AbstractSearchRequest,
+				types.AbstractSearchResult,
+				D
+		  >;
+	local?: boolean;
+	resolve?: Resolve;
+	signal?: AbortSignal;
+	waitFor?: number; // how long to wait for a non-empty result set
+};
+
 export type SearchOptions<
 	R,
 	D,
@@ -228,6 +245,8 @@ function isSubclassOf(
 	return false;
 }
 
+const DEFAULT_TIMEOUT = 1e4;
+
 const DEFAULT_INDEX_BY = "id";
 
 export type CanSearch = (
@@ -298,6 +317,7 @@ export type OpenOptions<
 	I,
 	D extends ReplicationDomain<any, Operation, any>,
 > = {
+	documentEvents: TypedEventTarget<DocumentEvents<T, I>>;
 	documentType: AbstractType<T>;
 	dbType: AbstractType<types.IDocumentStore<T>>;
 	log: SharedLog<Operation, D, any>;
@@ -399,6 +419,9 @@ export class DocumentIndex<
 	private _maybeOpen: (value: T & Program) => Promise<T & Program>;
 	private canSearch?: CanSearch;
 	private canRead?: CanRead<I>;
+
+	private documentEvents: TypedEventTarget<DocumentEvents<T, I>>;
+
 	private _joinListener?: (e: { detail: PublicSignKey }) => Promise<void>;
 
 	private _resultQueue: Map<
@@ -455,7 +478,7 @@ export class DocumentIndex<
 		this.indexedTypeIsDocumentType =
 			!properties.transform?.type ||
 			properties.transform?.type === properties.documentType;
-
+		this.documentEvents = properties.documentEvents;
 		this.compatibility = properties.compatibility;
 		this.canRead = properties.canRead;
 		this.canSearch = properties.canSearch;
@@ -791,26 +814,58 @@ export class DocumentIndex<
 		return dropped;
 	}
 
-	public async get<Options extends QueryOptions<T, D, true | undefined>>(
+	public async get<Options extends GetOptions<T, D, true | undefined>>(
 		key: indexerTypes.Ideable | indexerTypes.IdKey,
 		options?: Options,
 	): Promise<WithIndexedContext<T, I>>;
 
-	public async get<Options extends QueryOptions<T, D, false>>(
+	public async get<Options extends GetOptions<T, D, false>>(
 		key: indexerTypes.Ideable | indexerTypes.IdKey,
 		options?: Options,
 	): Promise<WithContext<I>>;
 
 	public async get<
-		Options extends QueryOptions<T, D, Resolve>,
+		Options extends GetOptions<T, D, Resolve>,
 		Resolve extends boolean | undefined = ExtractResolveFromOptions<Options>,
 	>(key: indexerTypes.Ideable | indexerTypes.IdKey, options?: Options) {
-		const result = (
-			await this.getDetailed(
-				key instanceof indexerTypes.IdKey ? key : indexerTypes.toId(key),
-				options,
-			)
-		)?.[0]?.results[0];
+		let idKey =
+			key instanceof indexerTypes.IdKey ? key : indexerTypes.toId(key);
+		const result = (await this.getDetailed(idKey, options))?.[0]?.results[0];
+
+		// if no results, and we have remote joining options, we wait for the timout and if there are joining peers we re-query
+		if (!result) {
+			if (options?.waitFor) {
+				let deferred = pDefer<WithIndexedContext<T, I> | WithContext<I>>();
+
+				const listener = (evt: CustomEvent<DocumentsChange<T, I>>) => {
+					for (const added of evt.detail.added) {
+						const id = this.indexByResolver(added.__indexed);
+						if (id === idKey.primitive) {
+							deferred.resolve(added);
+						}
+					}
+				};
+
+				this.documentEvents.addEventListener("change", listener);
+
+				let cleanup = () => {
+					this.documentEvents.removeEventListener("change", listener);
+					clearTimeout(timeout);
+					this.events.removeEventListener("close", resolveUndefined);
+				};
+
+				let resolveUndefined = () => {
+					cleanup();
+					deferred.resolve(undefined);
+				};
+
+				let timeout = setTimeout(resolveUndefined, options.waitFor);
+				this.events.addEventListener("close", resolveUndefined);
+
+				return deferred.promise;
+			}
+			return undefined;
+		}
 		return result?.value;
 	}
 
@@ -820,6 +875,13 @@ export class DocumentIndex<
 		await iterator.close();
 		return one[0];
 	}
+
+	public async getFromHash(hash: string) {
+		const iterator = this.index.iterate({ query: { hash } });
+		const one = await iterator.next(1);
+		await iterator.close();
+		return one[0];
+	}
 	public async put(
 		value: T,
 		id: indexerTypes.IdKey,
@@ -1243,10 +1305,10 @@ export class DocumentIndex<
 		}
 	}
 
-	async processCloseIteratorRequest(
+	processCloseIteratorRequest(
 		query: types.CloseIteratorRequest,
 		publicKey: PublicSignKey,
-	): Promise<void> {
+	): void {
 		const queueData = this._resultQueue.get(query.idString);
 		if (queueData && !queueData.from.equals(publicKey)) {
 			logger.info("Ignoring close iterator request from different peer");
@@ -1326,6 +1388,7 @@ export class DocumentIndex<
 				: await this._log.getCover(remote.domain ?? { args: undefined }, {
 						roleAge: remote.minAge,
 						eager: remote.eager,
+						reachableOnly: !!remote.joining, // when we want to merge joining we can ignore pending to be online peers and instead consider them once they become online
 					});
 
 			if (replicatorGroups) {
@@ -1682,8 +1745,8 @@ export class DocumentIndex<
 			let t0 = +new Date();
 			let waitForTime =
 				typeof options.remote.joining === "boolean"
-					? 1e4
-					: (options.remote.joining.waitFor ?? 1e4);
+					? DEFAULT_TIMEOUT
+					: (options.remote.joining.waitFor ?? DEFAULT_TIMEOUT);
 			let setDoneIfTimeout = false;
 			maybeSetDone = () => {
 				if (t0 + waitForTime < +new Date()) {
@@ -2128,15 +2191,24 @@ export class DocumentIndex<
 			return dedup(coercedBatch, this.indexByResolver);
 		};
 
-		let close = async () => {
+		let cleanupAndDone = () => {
 			cleanup();
-			done = true;
 			controller.abort(new AbortError("Iterator closed"));
+			this.prefetch?.accumulator.clear(queryRequestCoerced);
+			this.processCloseIteratorRequest(
+				queryRequestCoerced,
+				this.node.identity.publicKey,
+			);
+			done = true;
+		};
 
+		let close = async () => {
+			cleanupAndDone();
+
+			// send close to remote
 			const closeRequest = new types.CloseIteratorRequest({
 				id: queryRequestCoerced.id,
 			});
-			this.prefetch?.accumulator.clear(queryRequestCoerced);
 			const promises: Promise<any>[] = [];
 
 			for (const [peer, buffer] of peerBufferMap) {
@@ -2144,15 +2216,7 @@ export class DocumentIndex<
 					peerBufferMap.delete(peer);
 					continue;
 				}
-				// Fetch locally?
-				if (peer === this.node.identity.publicKey.hashcode()) {
-					promises.push(
-						this.processCloseIteratorRequest(
-							closeRequest,
-							this.node.identity.publicKey,
-						),
-					);
-				} else {
+				if (peer !== this.node.identity.publicKey.hashcode()) {
 					// Close remote
 					promises.push(
 						this._query.send(closeRequest, {
@@ -2172,6 +2236,11 @@ export class DocumentIndex<
 
 		let joinListener: ((e: { detail: PublicSignKey }) => void) | undefined;
 
+		let updateDeferred: ReturnType<typeof pDefer> | undefined;
+		const signalUpdate = () => updateDeferred?.resolve();
+		const waitForUpdate = () =>
+			updateDeferred ? updateDeferred.promise : Promise.resolve();
+
 		if (typeof options?.remote === "object" && options?.remote.joining) {
 			let onMissedResults =
 				typeof options?.remote?.joining === "object" &&
@@ -2179,7 +2248,38 @@ export class DocumentIndex<
 					? options.remote.joining.onMissedResults
 					: undefined;
 
+			updateDeferred = pDefer<void>();
+
+			const waitForTime =
+				typeof options.remote.joining === "object" &&
+				options.remote.joining.waitFor;
+
+			const prevMaybeSetDone = maybeSetDone;
+			maybeSetDone = () => {
+				prevMaybeSetDone();
+				if (done) signalUpdate(); // break deferred waits
+			};
+
+			let joinTimeoutId =
+				waitForTime &&
+				setTimeout(() => {
+					signalUpdate();
+				}, waitForTime);
+			controller.signal.addEventListener("abort", () => signalUpdate());
+
+			let activeJoins = new Set<string>();
+
 			joinListener = async (e: { detail: PublicSignKey }) => {
+				if (done) return;
+				const pk = e.detail;
+				const hash = pk.hashcode();
+
+				if (hash === this.node.identity.publicKey.hashcode()) return;
+				if (peerBufferMap.has(hash)) return;
+				if (activeJoins.has(hash)) return;
+
+				activeJoins.add(hash);
+
 				if (totalFetchedCounter > 0) {
 					// wait for the node to become a replicator, then so query
 					await this._log
@@ -2237,9 +2337,13 @@ export class DocumentIndex<
 									}
 								}
 							}
+							signalUpdate();
 						})
 						.catch(() => {
 							/* TODO error handling */
+						})
+						.finally(() => {
+							activeJoins.delete(hash);
 						});
 				}
 			};
@@ -2247,6 +2351,9 @@ export class DocumentIndex<
 			const cleanupDefault = cleanup;
 			cleanup = () => {
 				this._query.events.removeEventListener("join", joinListener!);
+				joinTimeoutId && clearTimeout(joinTimeoutId);
+				updateDeferred?.resolve();
+				updateDeferred = undefined;
 				return cleanupDefault();
 			};
 		}
@@ -2264,10 +2371,27 @@ export class DocumentIndex<
 			},
 			all: async () => {
 				let result: ValueTypeFromRequest<Resolve, T, I>[] = [];
+				let c = 0;
 				while (doneFn() !== true) {
+					c++;
 					let batch = await next(100);
-					result.push(...batch);
+					if (c > 100) {
+						break;
+					}
+					if (batch.length > 0) {
+						result.push(...batch);
+						continue;
+					}
+
+					// wait until: join fetch adds results, cleanup runs, or the join-wait times out
+					await waitForUpdate();
+
+					// re-arm the deferred for the next cycle (only if joining is enabled and we’re not done)
+					if (updateDeferred && !doneFn()) {
+						updateDeferred = pDefer<void>();
+					}
 				}
+				cleanupAndDone();
 				return result;
 			},
 		};