@peerbit/document 9.12.17 → 9.13.1-0fddff8

package/src/search.ts

@@ -1,10 +1,10 @@
 import { type AbstractType, field, serialize, variant } from "@dao-xyz/borsh";
 import type { PeerId, TypedEventTarget } from "@libp2p/interface";
+import type { Multiaddr } from "@multiformats/multiaddr";
 import { Cache } from "@peerbit/cache";
 import {
 	type MaybePromise,
 	PublicSignKey,
-	getPublicKeyFromPeerId,
 	sha256Base64Sync,
 } from "@peerbit/crypto";
 import * as types from "@peerbit/document-interface";
@@ -26,8 +26,12 @@ import {
 	type ReplicationDomain,
 	SharedLog,
 } from "@peerbit/shared-log";
-import { DataMessage, SilentDelivery } from "@peerbit/stream-interface";
-import { AbortError, waitFor } from "@peerbit/time";
+import {
+	DataMessage,
+	type PeerRefs,
+	SilentDelivery,
+} from "@peerbit/stream-interface";
+import { AbortError, TimeoutError, waitFor } from "@peerbit/time";
 import pDefer, { type DeferredPromise } from "p-defer";
 import { concat, fromString } from "uint8arrays";
 import { copySerialization } from "./borsh.js";
@@ -40,6 +44,8 @@ import { Prefetch } from "./prefetch.js";
 import type { ExtractArgs } from "./program.js";
 import { ResumableIterators } from "./resumable-iterator.js";
 
+const WARNING_WHEN_ITERATING_FOR_MORE_THAN = 1e5;
+
 const logger = loggerFn({ module: "document-index" });
 
 type BufferedResult<T, I extends Record<string, any>> = {
@@ -49,6 +55,102 @@ type BufferedResult<T, I extends Record<string, any>> = {
 	from: PublicSignKey;
 };
 
+export type UpdateMergeStrategy<
+	T,
+	I,
+	Resolve extends boolean | undefined,
+	RT = ValueTypeFromRequest<Resolve, T, I>,
+> =
+	| boolean
+	| {
+			filter?: (
+				evt: DocumentsChange<T, I>,
+			) => MaybePromise<DocumentsChange<T, I> | void>;
+	  };
+export type UpdateCallbacks<
+	T,
+	I,
+	Resolve extends boolean | undefined,
+	RT = ValueTypeFromRequest<Resolve, T, I>,
+> = {
+	/**
+	 * Fires on raw DB change events (added/removed/updated).
+	 * Use if you want low-level inspection of change streams.
+	 */
+	onChange?: (change: DocumentsChange<T, I>) => void | Promise<void>;
+
+	/**
+	 * Fires whenever the iterator yields a batch to the consumer.
+	 * Good for external sync (e.g. React state).
+	 */
+	onResults?: (
+		batch: RT[],
+		meta: { reason: "initial" | "next" | "join" | "change" },
+	) => void | Promise<void>;
+};
+
+/**
+ * Unified update options for iterate()/search()/get() and hooks.
+ * If you pass `true`, defaults to `{ merge: "sorted" }`.
+ */
+export type UpdateOptions<T, I, Resolve extends boolean | undefined> =
+	| boolean
+	| ({
+			/** Live update behavior. Only sorted merging is supported; optional filter can mutate/ignore events. */
+			merge?: UpdateMergeStrategy<T, I, Resolve>;
+	  } & UpdateCallbacks<T, I, Resolve>);
+
+export type JoiningTargets = {
+	/** Specific peers you care about */
+	peers?: Array<PublicSignKey | PeerId | string>; // string = hash or peer id
+
+	/** Multiaddrs you care about */
+	multiaddrs?: (string | Multiaddr)[];
+
+	/**
+	 * From the previous cover set (what you "knew" about earlier).
+	 * - "any": wait until at least 1 of the known peers is ready
+	 * - "all": wait until all known peers are ready
+	 * - number: wait until N known peers are ready
+	 */
+	known?: "any" | "all" | number;
+};
+
+export type JoiningTimeoutPolicy = "proceed" | "error";
+
+export type JoiningOnMissedResults = (evt: {
+	/** How many items should have preceded the current frontier. */
+	amount: number;
+
+	/** The peer whose arrival triggered the gap calculation. */
+	peer: PublicSignKey;
+}) => void | Promise<void>;
+
+export type LateResultsEvent = {
+	/** Count of items that should have appeared earlier than the current frontier */
+	amount: number;
+
+	/** If attributable, the peer that produced the late items */
+	peer?: PublicSignKey;
+};
+
+export type WaitBehavior =
+	| "block" // hold the *first* fetch until readiness condition is met or timeout
+	| "keep-open"; // return immediately; iterator stays open listening for late peers
+
+export type WaitPolicy = {
+	timeout: number; // max time to wait
+	until?: "any"; // readiness condition, TODO more options like "cover" (to wait for this.log.watiForReplicators)
+	onTimeout?: "proceed" | "error"; // proceed = continue with whoever's ready
+	behavior?: WaitBehavior; // default: "keep-open"
+};
+
+export type ReachScope = {
+	/** who to consider for readiness */
+	eager?: boolean; // not yet matured
+	discover?: PublicSignKey[]; // wait for these peers to be ready, assumes they are already in the dialqueue or connected, but not actively subscribing yet
+};
+
 export type RemoteQueryOptions<Q, R, D> = RPCRequestAllOptions<Q, R> & {
 	replicate?: boolean;
 	minAge?: number;
@@ -61,12 +163,15 @@ export type RemoteQueryOptions<Q, R, D> = RPCRequestAllOptions<Q, R> & {
 		| {
 				range: CoverRange<number | bigint>;
 		  };
-	eager?: boolean; // whether to query newly joined peers before they have matured
-	joining?:
-		| boolean
-		| { waitFor?: number; onMissedResults?: (evt: { amount: number }) => void }; // whether to query peers that are joining the network
+	/** WHO can answer? How do we grow the candidate set? */
+	reach?: ReachScope;
+	/** WHEN are we allowed to proceed? Quorum semantics over a chosen group. */
+	wait?: WaitPolicy;
+
+	onLateResults?: (evt: LateResultsEvent) => void | Promise<void>;
 };
-export type QueryOptions<R, D, Resolve extends boolean | undefined> = {
+
+export type QueryOptions<T, I, D, Resolve extends boolean | undefined> = {
 	remote?:
 		| boolean
 		| RemoteQueryOptions<
@@ -77,9 +182,16 @@ export type QueryOptions<R, D, Resolve extends boolean | undefined> = {
 	local?: boolean;
 	resolve?: Resolve;
 	signal?: AbortSignal;
+	updates?: UpdateOptions<T, I, Resolve>;
+	/**
+	 * Controls iterator liveness after batches are consumed.
+	 * - 'onEmpty' (default): close when no more results
+	 * - 'manual': keep open until iterator.close() or program close; good for live updates
+	 */
+	closePolicy?: "onEmpty" | "manual";
 };
 
-export type GetOptions<R, D, Resolve extends boolean | undefined> = {
+export type GetOptions<T, I, D, Resolve extends boolean | undefined> = {
 	remote?:
 		| boolean
 		| RemoteQueryOptions<
@@ -94,10 +206,11 @@ export type GetOptions<R, D, Resolve extends boolean | undefined> = {
 };
 
 export type SearchOptions<
-	R,
+	T,
+	I,
 	D,
 	Resolve extends boolean | undefined,
-> = QueryOptions<R, D, Resolve>;
+> = QueryOptions<T, I, D, Resolve>;
 
 type Transformer<T, I> = (obj: T, context: types.Context) => MaybePromise<I>;
 
@@ -108,13 +221,15 @@ export type ResultsIterator<T> = {
 	all: () => Promise<T[]>;
 	pending: () => number | undefined;
 	first: () => Promise<T | undefined>;
+	[Symbol.asyncIterator]: () => AsyncIterator<T>;
 };
 
 type QueryDetailedOptions<
 	T,
+	I,
 	D,
 	Resolve extends boolean | undefined,
-> = QueryOptions<T, D, Resolve> & {
+> = QueryOptions<T, I, D, Resolve> & {
 	onResponse?: (
 		response: types.AbstractSearchResult,
 		from: PublicSignKey,
@@ -122,6 +237,7 @@ type QueryDetailedOptions<
 	remote?: {
 		from?: string[]; // if specified, only query these peers
 	};
+	fetchFirstForRemote?: Set<string>;
 };
 
 type QueryLike = {
@@ -130,7 +246,7 @@ type QueryLike = {
 };
 
 type ExtractResolveFromOptions<O> =
-	O extends QueryOptions<any, any, infer X>
+	O extends QueryOptions<any, any, any, infer X>
 		? X extends boolean // if X is a boolean (true or false)
 			? X
 			: true // else default to true
@@ -138,7 +254,7 @@ type ExtractResolveFromOptions<O> =
 
 const coerceQuery = <Resolve extends boolean | undefined>(
 	query: types.SearchRequest | types.SearchRequestIndexed | QueryLike,
-	options?: QueryOptions<any, any, Resolve>,
+	options?: QueryOptions<any, any, any, Resolve>,
 ) => {
 	let replicate =
 		typeof options?.remote !== "boolean" ? options?.remote?.replicate : false;
@@ -180,7 +296,7 @@ const introduceEntries = async <
 	documentType: AbstractType<T>,
 	indexedType: AbstractType<I>,
 	sync: (request: R, response: types.Results<any>) => Promise<void>,
-	options?: QueryDetailedOptions<T, D, any>,
+	options?: QueryDetailedOptions<T, I, D, any>,
 ): Promise<
 	RPCResponse<types.Results<types.ResultTypeFromRequest<R, T, I>>>[]
 > => {
@@ -815,18 +931,18 @@ export class DocumentIndex<
 		return dropped;
 	}
 
-	public async get<Options extends GetOptions<T, D, true | undefined>>(
+	public async get<Options extends GetOptions<T, I, D, true | undefined>>(
 		key: indexerTypes.Ideable | indexerTypes.IdKey,
 		options?: Options,
 	): Promise<WithIndexedContext<T, I>>;
 
-	public async get<Options extends GetOptions<T, D, false>>(
+	public async get<Options extends GetOptions<T, I, D, false>>(
 		key: indexerTypes.Ideable | indexerTypes.IdKey,
 		options?: Options,
 	): Promise<WithContext<I>>;
 
 	public async get<
-		Options extends GetOptions<T, D, Resolve>,
+		Options extends GetOptions<T, I, D, Resolve>,
 		Resolve extends boolean | undefined = ExtractResolveFromOptions<Options>,
 	>(key: indexerTypes.Ideable | indexerTypes.IdKey, options?: Options) {
 		let deferred:
@@ -879,16 +995,15 @@ export class DocumentIndex<
 						? { ...options.remote }
 						: {};
 			if (baseRemote) {
-				const prevJoining = baseRemote.joining;
+				const waitPolicy = baseRemote.wait;
 				if (
-					!prevJoining ||
-					prevJoining === true ||
-					(typeof prevJoining === "object" &&
-						(prevJoining.waitFor || 0) < options.waitFor)
+					!waitPolicy ||
+					(typeof waitPolicy === "object" &&
+						(waitPolicy.timeout || 0) < options.waitFor)
 				) {
-					baseRemote.joining = {
-						...(typeof prevJoining === "object" ? prevJoining : {}),
-						waitFor: options.waitFor,
+					baseRemote.wait = {
+						...(typeof waitPolicy === "object" ? waitPolicy : {}),
+						timeout: options.waitFor,
 					};
 				}
 			}
@@ -897,10 +1012,11 @@ export class DocumentIndex<
 			let joinListener: (() => void) | undefined;
 			if (baseRemote) {
 				joinListener = this.attachJoinListener({
+					eager: baseRemote.reach?.eager,
 					onPeer: async (pk) => {
 						if (cleanedUp) return;
 						const hash = pk.hashcode();
-						const requeryOptions: QueryOptions<T, D, Resolve> = {
+						const requeryOptions: QueryOptions<T, I, D, Resolve> = {
 							...(options as any),
 							remote: {
 								...(baseRemote || {}),
@@ -1005,14 +1121,14 @@ export class DocumentIndex<
 	}
 
 	public async getDetailed<
-		Options extends QueryOptions<T, D, Resolve>,
+		Options extends QueryOptions<T, I, D, Resolve>,
 		Resolve extends boolean | undefined = ExtractResolveFromOptions<Options>,
 		RT extends types.Result = Resolve extends true
 			? types.ResultValue<WithIndexedContext<T, I>>
 			: types.ResultIndexedValue<WithContext<I>>,
 	>(
 		key: indexerTypes.IdKey | indexerTypes.IdPrimitive,
-		options?: QueryOptions<T, D, Resolve>,
+		options?: QueryOptions<T, I, D, Resolve>,
 	): Promise<types.Results<RT>[] | undefined> {
 		let coercedOptions = options;
 		if (options?.remote && typeof options.remote !== "boolean") {
@@ -1049,7 +1165,7 @@ export class DocumentIndex<
 						new indexerTypes.ByteMatchQuery({ key: this.indexBy, value: key }),
 					],
 				}),
-				coercedOptions,
+				coercedOptions as QueryDetailedOptions<T, I, D, boolean | undefined>,
 			);
 		} else {
 			const indexableKey = indexerTypes.toIdeable(key);
@@ -1068,7 +1184,7 @@ export class DocumentIndex<
 							}),
 						],
 					}),
-					coercedOptions,
+					coercedOptions as QueryDetailedOptions<T, I, D, boolean | undefined>,
 				);
 			} else if (typeof indexableKey === "string") {
 				results = await this.queryCommence(
@@ -1080,7 +1196,7 @@ export class DocumentIndex<
 							}),
 						],
 					}),
-					coercedOptions,
+					coercedOptions as QueryDetailedOptions<T, I, D, boolean | undefined>,
 				);
 			} else if (indexableKey instanceof Uint8Array) {
 				results = await this.queryCommence(
@@ -1092,7 +1208,7 @@ export class DocumentIndex<
 							}),
 						],
 					}),
-					coercedOptions,
+					coercedOptions as QueryDetailedOptions<T, I, D, boolean | undefined>,
 				);
 			}
 		}
@@ -1364,10 +1480,131 @@ export class DocumentIndex<
 		}
 	}
 
+	private async waitForCoverReady(params: {
+		domain?: { args?: ExtractArgs<D> } | { range: CoverRange<number | bigint> };
+		eager?: boolean;
+		settle: "any";
+		timeout: number;
+		signal?: AbortSignal;
+		onTimeout?: "proceed" | "error";
+	}) {
+		const {
+			domain,
+			eager,
+			settle,
+			timeout,
+			signal,
+			onTimeout = "proceed",
+		} = params;
+
+		if (settle !== "any") {
+			return;
+		}
+
+		const properties =
+			domain && "range" in domain
+				? { range: domain.range }
+				: { args: domain?.args };
+		const selfHash = this.node.identity.publicKey.hashcode();
+
+		const ready = async () => {
+			const cover = await this._log.getCover(properties, { eager });
+			return cover.some((hash) => hash !== selfHash);
+		};
+
+		if (await ready()) {
+			return;
+		}
+
+		const deferred = pDefer<void>();
+		let settled = false;
+		let cleaned = false;
+		let timer: ReturnType<typeof setTimeout> | undefined;
+		let checking = false;
+
+		const cleanup = () => {
+			if (cleaned) {
+				return;
+			}
+			cleaned = true;
+			this._log.events.removeEventListener("replicator:join", onEvent);
+			this._log.events.removeEventListener("replication:change", onEvent);
+			this._log.events.removeEventListener("replicator:mature", onEvent);
+			signal?.removeEventListener("abort", onAbort);
+			if (timer != null) {
+				clearTimeout(timer);
+				timer = undefined;
+			}
+		};
+
+		const resolve = () => {
+			if (settled) {
+				return;
+			}
+			settled = true;
+			cleanup();
+			deferred.resolve();
+		};
+
+		const reject = (error: Error) => {
+			if (settled) {
+				return;
+			}
+			settled = true;
+			cleanup();
+			deferred.reject(error);
+		};
+
+		const onAbort = () => reject(new AbortError());
+
+		const onEvent = async () => {
+			if (checking) {
+				return;
+			}
+			checking = true;
+			try {
+				if (await ready()) {
+					resolve();
+				}
+			} catch (error) {
+				reject(error instanceof Error ? error : new Error(String(error)));
+			} finally {
+				checking = false;
+			}
+		};
+
+		if (signal) {
+			signal.addEventListener("abort", onAbort);
+		}
+
+		if (timeout > 0) {
+			timer = setTimeout(() => {
+				if (onTimeout === "error") {
+					reject(
+						new TimeoutError("Timeout waiting for participating replicator"),
+					);
+				} else {
+					resolve();
+				}
+			}, timeout);
+		}
+
+		this._log.events.addEventListener("replicator:join", onEvent);
+		this._log.events.addEventListener("replication:change", onEvent);
+		this._log.events.addEventListener("replicator:mature", onEvent);
+
+		try {
+			await deferred.promise;
+		} finally {
+			cleanup();
+		}
+	}
+
 	// Utility: attach a join listener that waits until a peer is a replicator,
 	// then invokes the provided callback. Returns a detach function.
 	private attachJoinListener(params: {
 		signal?: AbortSignal;
+		eager?: boolean;
 		onPeer: (pk: PublicSignKey) => Promise<void> | void;
 	}): () => void {
 		const active = new Set<string>();
@@ -1382,6 +1619,7 @@ export class DocumentIndex<
 				await this._log
 					.waitForReplicator(pk, {
 						signal: params.signal,
+						eager: params.eager,
 					})
 					.catch(() => undefined);
 				if (params.signal?.aborted) return;
@@ -1419,7 +1657,8 @@ export class DocumentIndex<
 		RT extends types.Result = types.ResultTypeFromRequest<R, T, I>,
 	>(
 		queryRequest: R,
-		options?: QueryDetailedOptions<T, D, boolean | undefined>,
+		options?: QueryDetailedOptions<T, I, D, boolean | undefined>,
+		fetchFirstForRemote?: Set<string>,
 	): Promise<types.Results<RT>[]> {
 		const local = typeof options?.local === "boolean" ? options?.local : true;
 		let remote:
@@ -1477,8 +1716,8 @@ export class DocumentIndex<
 				? options?.remote?.from
 				: await this._log.getCover(remote.domain ?? { args: undefined }, {
 						roleAge: remote.minAge,
-						eager: remote.eager,
-						reachableOnly: !!remote.joining, // when we want to merge joining we can ignore pending to be online peers and instead consider them once they become online
+						eager: remote.reach?.eager,
+						reachableOnly: !!remote.wait, // when we want to merge joining we can ignore pending to be online peers and instead consider them once they become online
 					});
 
 			if (replicatorGroups) {
@@ -1508,6 +1747,13 @@ export class DocumentIndex<
 						if (hash === this.node.identity.publicKey.hashcode()) {
 							return false;
 						}
+
+						if (fetchFirstForRemote?.has(hash)) {
+							// we already fetched this one for remote, no need to do it again
+							return false;
+						}
+						fetchFirstForRemote?.add(hash);
+
 						const resultAlready = this._prefetch?.accumulator.consume(
 							queryRequest,
 							hash,
@@ -1637,11 +1883,11 @@ export class DocumentIndex<
 
 	public search(
 		queryRequest: QueryLike,
-		options?: SearchOptions<T, D, true>,
+		options?: SearchOptions<T, I, D, true>,
 	): Promise<ValueTypeFromRequest<true, T, I>[]>;
 	public search(
 		queryRequest: QueryLike,
-		options?: SearchOptions<T, D, false>,
+		options?: SearchOptions<T, I, D, false>,
 	): Promise<ValueTypeFromRequest<false, T, I>[]>;
 
 	/**
@@ -1652,11 +1898,11 @@ export class DocumentIndex<
 	 */
 	public async search<
 		R extends types.SearchRequest | types.SearchRequestIndexed | QueryLike,
-		O extends SearchOptions<T, D, Resolve>,
-		Resolve extends boolean | undefined = ExtractResolveFromOptions<O>,
+		O extends SearchOptions<T, I, D, Resolve>,
+		Resolve extends boolean = ExtractResolveFromOptions<O>,
 	>(
 		queryRequest: R,
-		options?: SearchOptions<T, D, Resolve>,
+		options?: O,
 	): Promise<ValueTypeFromRequest<Resolve, T, I>[]> {
 		// Set fetch to search size, or max value (default to max u32 (4294967295))
 		const coercedRequest: types.SearchRequest | types.SearchRequestIndexed =
@@ -1664,7 +1910,7 @@ export class DocumentIndex<
 		coercedRequest.fetch = coercedRequest.fetch ?? 0xffffffff;
 
 		// So that the iterator is pre-fetching the right amount of entries
-		const iterator = this.iterate(coercedRequest, options);
+		const iterator = this.iterate<Resolve>(coercedRequest, options);
 
 		// So that this call will not do any remote requests
 		const allResults: ValueTypeFromRequest<Resolve, T, I>[] = [];
@@ -1729,11 +1975,11 @@ export class DocumentIndex<
 
 	public iterate(
 		query?: QueryLike,
-		options?: QueryOptions<T, D, undefined>,
+		options?: QueryOptions<T, I, D, undefined>,
 	): ResultsIterator<ValueTypeFromRequest<true, T, I>>;
 	public iterate<Resolve extends boolean>(
 		query?: QueryLike,
-		options?: QueryOptions<T, D, Resolve>,
+		options?: QueryOptions<T, I, D, Resolve>,
 	): ResultsIterator<ValueTypeFromRequest<Resolve, T, I>>;
 
 	/**
@@ -1744,11 +1990,11 @@ export class DocumentIndex<
 	 */
 	public iterate<
 		R extends types.SearchRequest | types.SearchRequestIndexed | QueryLike,
-		O extends SearchOptions<T, D, Resolve>,
+		O extends SearchOptions<T, I, D, Resolve>,
 		Resolve extends boolean | undefined = ExtractResolveFromOptions<O>,
 	>(
 		queryRequest?: R,
-		options?: QueryOptions<T, D, Resolve>,
+		options?: QueryOptions<T, I, D, Resolve>,
 	): ResultsIterator<ValueTypeFromRequest<Resolve, T, I>> {
 		if (
 			queryRequest instanceof types.SearchRequest &&
@@ -1803,6 +2049,7 @@ export class DocumentIndex<
 		const visited = new Set<string | number | bigint>();
 
 		let done = false;
+		let drain = false; // if true, close on empty once (overrides manual)
 		let first = false;
 
 		// TODO handle join/leave while iterating
@@ -1827,147 +2074,186 @@ export class DocumentIndex<
 			return [...peerBufferMap.values()].map((x) => x.buffer).flat();
 		};
 
-		let maybeSetDone: () => void;
-		let unsetDone: () => void;
-		let cleanup = () => {};
-
-		if (typeof options?.remote === "object" && options.remote.joining) {
-			let t0 = +new Date();
-			let waitForTime =
-				typeof options.remote.joining === "boolean"
-					? DEFAULT_TIMEOUT
-					: (options.remote.joining.waitFor ?? DEFAULT_TIMEOUT);
-			let setDoneIfTimeout = false;
-			maybeSetDone = () => {
-				if (t0 + waitForTime < +new Date()) {
-					cleanup();
-					done = true;
-				} else {
-					setDoneIfTimeout = true;
-				}
-			};
-			unsetDone = () => {
-				setDoneIfTimeout = false;
-				done = false;
-			};
-			let timeout = setTimeout(() => {
-				if (setDoneIfTimeout) {
-					cleanup();
-					done = true;
-				}
-			}, waitForTime);
+		let maybeSetDone = () => {
+			cleanup();
+			done = true;
+		};
+		let unsetDone = () => {
+			cleanup();
+			done = false;
+		};
+		let cleanup = () => {
+			this.clearResultsQueue(queryRequestCoerced);
+		};
 
-			cleanup = () => {
-				this.clearResultsQueue(queryRequestCoerced);
-				clearTimeout(timeout);
-			};
-		} else {
-			maybeSetDone = () => {
-				cleanup();
-				done = true;
-			};
-			unsetDone = () => {
-				cleanup();
-				done = false;
-			};
-			cleanup = () => {
-				this.clearResultsQueue(queryRequestCoerced);
-			};
+		let warmupPromise: Promise<any> | undefined = undefined;
+
+		if (typeof options?.remote === "object") {
+			let waitForTime: number | undefined = undefined;
+			if (options.remote.wait) {
+				let t0 = +new Date();
+
+				waitForTime =
+					typeof options.remote.wait === "boolean"
+						? DEFAULT_TIMEOUT
+						: (options.remote.wait.timeout ?? DEFAULT_TIMEOUT);
+				let setDoneIfTimeout = false;
+				maybeSetDone = () => {
+					if (t0 + waitForTime! < +new Date()) {
+						cleanup();
+						done = true;
+					} else {
+						setDoneIfTimeout = true;
+					}
+				};
+				unsetDone = () => {
+					setDoneIfTimeout = false;
+					done = false;
+				};
+				let timeout = setTimeout(() => {
+					if (setDoneIfTimeout) {
+						cleanup();
+						done = true;
+					}
+				}, waitForTime);
+
+				cleanup = () => {
+					this.clearResultsQueue(queryRequestCoerced);
+					clearTimeout(timeout);
+				};
+			}
+
+			if (options.remote.reach?.discover) {
+				warmupPromise = this.waitFor(options.remote.reach.discover, {
+					signal: controller.signal,
+					seek: "present",
+					timeout: waitForTime ?? DEFAULT_TIMEOUT,
+				});
+				options.remote.reach.eager = true; // include the results from the discovered peer even if it is not mature
+			}
+
+			const waitPolicy =
+				typeof options.remote.wait === "object"
+					? options.remote.wait
+					: undefined;
+			if (
+				waitPolicy?.behavior === "block" &&
+				(waitPolicy.until ?? "any") === "any"
+			) {
+				const blockPromise = this.waitForCoverReady({
+					domain: options.remote.domain,
+					eager: options.remote.reach?.eager,
+					settle: "any",
+					timeout: waitPolicy.timeout ?? DEFAULT_TIMEOUT,
+					signal: controller.signal,
+					onTimeout: waitPolicy.onTimeout,
+				});
+				warmupPromise = warmupPromise
+					? Promise.all([warmupPromise, blockPromise]).then(() => undefined)
+					: blockPromise;
+			}
 		}
 
 		const fetchFirst = async (
 			n: number,
-			fetchOptions?: { from?: string[] },
+			fetchOptions?: { from?: string[]; fetchedFirstForRemote?: Set<string> },
 		): Promise<boolean> => {
+			await warmupPromise;
 			let hasMore = false;
 
 			queryRequestCoerced.fetch = n;
-			await this.queryCommence(queryRequestCoerced, {
-				local: fetchOptions?.from != null ? false : options?.local,
-				remote:
-					options?.remote !== false
-						? {
-								...(typeof options?.remote === "object" ? options.remote : {}),
-								from: fetchOptions?.from,
-							}
-						: false,
-				resolve,
-				onResponse: async (response, from) => {
-					if (!from) {
-						logger.error("Missing response from");
-						return;
-					}
-					if (response instanceof types.NoAccess) {
-						logger.error("Dont have access");
-						return;
-					} else if (response instanceof types.Results) {
-						const results = response as types.Results<
-							types.ResultTypeFromRequest<R, T, I>
-						>;
-
-						if (results.kept === 0n && results.results.length === 0) {
+			await this.queryCommence(
+				queryRequestCoerced,
+				{
+					local: fetchOptions?.from != null ? false : options?.local,
+					remote:
+						options?.remote !== false
+							? {
+									...(typeof options?.remote === "object"
+										? options.remote
+										: {}),
+									from: fetchOptions?.from,
+								}
+							: false,
+					resolve,
+					onResponse: async (response, from) => {
+						if (!from) {
+							logger.error("Missing response from");
 							return;
 						}
+						if (response instanceof types.NoAccess) {
+							logger.error("Dont have access");
+							return;
+						} else if (response instanceof types.Results) {
+							const results = response as types.Results<
+								types.ResultTypeFromRequest<R, T, I>
+							>;
 
-						if (results.kept > 0n) {
-							hasMore = true;
-						}
-						const buffer: BufferedResult<
-							types.ResultTypeFromRequest<R, T, I> | I,
-							I
-						>[] = [];
-
-						for (const result of results.results) {
-							if (result instanceof types.ResultValue) {
-								const indexKey = indexerTypes.toId(
-									this.indexByResolver(result.value),
-								).primitive;
-								if (visited.has(indexKey)) {
-									continue;
-								}
-								visited.add(indexKey);
-
-								buffer.push({
-									value: result.value as types.ResultTypeFromRequest<R, T, I>,
-									context: result.context,
-									from,
-									indexed: await this.resolveIndexed<R>(
-										result,
-										results.results,
-									),
-								});
-							} else {
-								const indexKey = indexerTypes.toId(
-									this.indexByResolver(result.value),
-								).primitive;
+							if (results.kept === 0n && results.results.length === 0) {
+								return;
+							}
 
-								if (visited.has(indexKey)) {
-									continue;
+							if (results.kept > 0n) {
+								hasMore = true;
+							}
+							const buffer: BufferedResult<
+								types.ResultTypeFromRequest<R, T, I> | I,
+								I
+							>[] = [];
+
+							for (const result of results.results) {
+								if (result instanceof types.ResultValue) {
+									const indexKey = indexerTypes.toId(
+										this.indexByResolver(result.value),
+									).primitive;
+									if (visited.has(indexKey)) {
+										continue;
+									}
+									visited.add(indexKey);
+
+									buffer.push({
+										value: result.value as types.ResultTypeFromRequest<R, T, I>,
+										context: result.context,
+										from,
+										indexed: await this.resolveIndexed<R>(
+											result,
+											results.results,
+										),
+									});
+								} else {
+									const indexKey = indexerTypes.toId(
+										this.indexByResolver(result.value),
+									).primitive;
+
+									if (visited.has(indexKey)) {
+										continue;
+									}
+									visited.add(indexKey);
+									buffer.push({
+										value: result.value,
+										context: result.context,
+										from,
+										indexed: coerceWithContext(
+											result.indexed || result.value,
+											result.context,
+										),
+									});
 								}
-								visited.add(indexKey);
-								buffer.push({
-									value: result.value,
-									context: result.context,
-									from,
-									indexed: coerceWithContext(
-										result.indexed || result.value,
-										result.context,
-									),
-								});
 							}
-						}
 
-						peerBufferMap.set(from.hashcode(), {
-							buffer,
-							kept: Number(response.kept),
-						});
-					} else {
-						throw new Error(
-							"Unsupported result type: " + response?.constructor?.name,
-						);
-					}
+							peerBufferMap.set(from.hashcode(), {
+								buffer,
+								kept: Number(response.kept),
+							});
+						} else {
+							throw new Error(
+								"Unsupported result type: " + response?.constructor?.name,
+							);
+						}
+					},
 				},
-			});
+				fetchOptions?.fetchedFirstForRemote,
+			);
 
 			if (!hasMore) {
 				maybeSetDone();
@@ -2015,7 +2301,10 @@ export class DocumentIndex<
 						amount: n - buffer.buffer.length,
 					});
 					// Fetch locally?
-					if (peer === this.node.identity.publicKey.hashcode()) {
+					if (
+						peer === this.node.identity.publicKey.hashcode() &&
+						this._resumableIterators.has(queryRequestCoerced.idString)
+					) {
 						promises.push(
 							this.processQuery(
 								collectRequest,
@@ -2116,7 +2405,7 @@ export class DocumentIndex<
 										this.documentType,
 										this.indexedType,
 										this._sync,
-										options,
+										options as QueryDetailedOptions<T, I, D, any>,
 									)
 										.then(async (responses) => {
 											return Promise.all(
@@ -2229,11 +2518,10 @@ export class DocumentIndex<
 				),
 			);
 
-			const pendingMoreResults = n < results.length;
-
 			lastValueInOrder = results[0] || lastValueInOrder;
-
+			const pendingMoreResults = n < results.length; // check if there are more results to fetch, before splicing
 			const batch = results.splice(0, n);
+			const hasMore = !fetchedAll || pendingMoreResults;
 
 			for (const result of batch) {
 				const arr = peerBufferMap.get(result.from.hashcode());
@@ -2247,7 +2535,6 @@ export class DocumentIndex<
 				}
 			}
 
-			const hasMore = !fetchedAll || pendingMoreResults;
 			if (hasMore) {
 				unsetDone();
 			} else {
@@ -2281,6 +2568,8 @@ export class DocumentIndex<
 				) as ValueTypeFromRequest<Resolve, T, I>[];
 			}
 
+			// no extra queued-first/last in simplified API
+
 			return dedup(coercedBatch, this.indexByResolver);
 		};
 
@@ -2329,26 +2618,141 @@ export class DocumentIndex<
 
 		let joinListener: (() => void) | undefined;
 
+		let fetchedFirstForRemote: Set<string> | undefined = undefined;
+
 		let updateDeferred: ReturnType<typeof pDefer> | undefined;
 		const signalUpdate = () => updateDeferred?.resolve();
-		const waitForUpdate = () =>
+		const _waitForUpdate = () =>
 			updateDeferred ? updateDeferred.promise : Promise.resolve();
 
-		if (typeof options?.remote === "object" && options?.remote.joining) {
+		// ---------------- Live updates wiring (sorted-only with optional filter) ----------------
+		const normalizeUpdatesOption = (
+			u?: UpdateOptions<T, I, Resolve>,
+		):
+			| {
+					merge?:
+						| {
+								filter?: (
+									evt: DocumentsChange<T, I>,
+								) => MaybePromise<DocumentsChange<T, I> | void>;
+						  }
+						| undefined;
+			  }
+			| undefined => {
+			if (u == null || u === false) return undefined;
+			if (u === true)
+				return {
+					merge: {
+						filter: (evt) => evt,
+					},
+				};
+			if (typeof u === "object") {
+				return {
+					merge: u.merge
+						? {
+								filter:
+									typeof u.merge === "object" ? u.merge.filter : (evt) => evt,
+							}
+						: {},
+				};
+			}
+			return undefined;
+		};
+
+		const mergePolicy = normalizeUpdatesOption(options?.updates);
+		const hasLiveUpdates = mergePolicy !== undefined;
+
+		// sorted-only mode: no per-queue handling
+
+		// If live updates enabled, ensure deferred exists so awaiting paths can block until changes
+		if (hasLiveUpdates && !updateDeferred) {
+			updateDeferred = pDefer<void>();
+		}
+
+		let updatesCleanup: (() => void) | undefined;
+		if (hasLiveUpdates) {
+			const localHash = this.node.identity.publicKey.hashcode();
+			if (mergePolicy?.merge) {
+				// Ensure local buffer exists for sorted merging
+				if (!peerBufferMap.has(localHash)) {
+					peerBufferMap.set(localHash, { kept: 0, buffer: [] });
+				}
+			}
+
+			const onChange = async (evt: CustomEvent<DocumentsChange<T, I>>) => {
+				// Optional filter to mutate/suppress change events
+				let filtered: DocumentsChange<T, I> | void = evt.detail;
+				if (mergePolicy?.merge?.filter) {
+					filtered = await mergePolicy.merge?.filter(evt.detail);
+				}
+				if (filtered) {
+					// Remove entries that were deleted from all pending structures
+					if (filtered.removed?.length) {
+						// Remove from peer buffers
+						for (const [_peer, entry] of peerBufferMap) {
+							entry.buffer = entry.buffer.filter((x) => {
+								const id = indexerTypes.toId(
+									this.indexByResolver(x.indexed),
+								).primitive;
+								return !filtered!.removed!.some(
+									(r) =>
+										indexerTypes.toId(this.indexByResolver(r.__indexed))
+											.primitive === id,
+								);
+							});
+						}
+						// no non-sorted queues in simplified mode
+					}
+
+					// Add new entries per strategy (sorted-only)
+					if (filtered.added?.length) {
+						const buf = peerBufferMap.get(localHash)!;
+						for (const added of filtered.added) {
+							const id = indexerTypes.toId(
+								this.indexByResolver(added.__indexed),
+							).primitive;
+							if (visited.has(id)) continue; // already presented
+							visited.add(id);
+							buf.buffer.push({
+								value: (resolve ? added : added.__indexed) as any,
+								context: added.__context,
+								from: this.node.identity.publicKey,
+								indexed: coerceWithContext(added.__indexed, added.__context),
+							});
+						}
+						buf.kept = buf.buffer.length;
+					}
+				}
+				typeof options?.updates === "object" &&
+					options?.updates?.onChange?.(evt.detail);
+				signalUpdate();
+			};
+
+			this.documentEvents.addEventListener("change", onChange);
+			updatesCleanup = () => {
+				this.documentEvents.removeEventListener("change", onChange);
+			};
+			const cleanupDefaultUpdates = cleanup;
+			cleanup = () => {
+				updatesCleanup?.();
+				return cleanupDefaultUpdates();
+			};
+		}
+
+		if (typeof options?.remote === "object" && options?.remote.wait) {
 			// was used to account for missed results when a peer joins; omitted in this minimal handler
 
 			updateDeferred = pDefer<void>();
 
 			// derive optional onMissedResults callback if provided
 			let onMissedResults =
-				typeof options?.remote?.joining === "object" &&
-				typeof options?.remote.joining.onMissedResults === "function"
-					? options.remote.joining.onMissedResults
+				typeof options?.remote?.wait === "object" &&
+				typeof options?.remote.onLateResults === "function"
+					? options.remote.onLateResults
 					: undefined;
 
 			const waitForTime =
-				typeof options.remote.joining === "object" &&
-				options.remote.joining.waitFor;
+				typeof options.remote.wait === "object" && options.remote.wait.timeout;
 
 			const prevMaybeSetDone = maybeSetDone;
 			maybeSetDone = () => {
@@ -2362,15 +2766,22 @@ export class DocumentIndex<
 					signalUpdate();
 				}, waitForTime);
 			controller.signal.addEventListener("abort", () => signalUpdate());
-
+			fetchedFirstForRemote = new Set<string>();
 			joinListener = this.attachJoinListener({
 				signal: controller.signal,
+				eager: options.remote.reach?.eager,
 				onPeer: async (pk) => {
 					if (done) return;
 					const hash = pk.hashcode();
+					await fetchPromise; // ensure fetches in flight are done
 					if (peerBufferMap.has(hash)) return;
+					if (fetchedFirstForRemote!.has(hash)) return;
 					if (totalFetchedCounter > 0) {
-						await fetchFirst(totalFetchedCounter, { from: [hash] });
+						fetchPromise = fetchFirst(totalFetchedCounter, {
+							from: [hash],
+							fetchedFirstForRemote,
+						});
+						await fetchPromise;
 						if (onMissedResults) {
 							const pending = peerBufferMap.get(hash)?.buffer;
 							if (pending && pending.length > 0) {
@@ -2409,6 +2820,29 @@ export class DocumentIndex<
 			};
 		}
 
+		if (options?.closePolicy === "manual") {
+			let prevMaybeSetDone = maybeSetDone;
+			maybeSetDone = () => {
+				if (drain) {
+					prevMaybeSetDone();
+				}
+			};
+		}
+		const remoteWaitActive =
+			typeof options?.remote === "object" && !!options.remote.wait;
+
+		const waitForUpdateAndResetDeferred = async () => {
+			if (remoteWaitActive) {
+				// wait until: join fetch adds results, cleanup runs, or the join-wait times out
+				await _waitForUpdate();
+
+				// re-arm the deferred for the next cycle (only if joining is enabled and we're not done)
+				if (updateDeferred && !doneFn()) {
+					updateDeferred = pDefer<void>();
+				}
+			}
+		};
+
 		return {
 			close,
 			next,
@@ -2421,26 +2855,24 @@ export class DocumentIndex<
 				return kept; // TODO this should be more accurate
 			},
 			all: async () => {
+				drain = true;
 				let result: ValueTypeFromRequest<Resolve, T, I>[] = [];
 				let c = 0;
 				while (doneFn() !== true) {
-					c++;
 					let batch = await next(100);
-					if (c > 100) {
-						break;
+					c += batch.length;
+					if (c > WARNING_WHEN_ITERATING_FOR_MORE_THAN) {
+						logger.warn(
+							"Iterating for more than " +
+								WARNING_WHEN_ITERATING_FOR_MORE_THAN +
+								" results",
+						);
 					}
 					if (batch.length > 0) {
 						result.push(...batch);
 						continue;
 					}
-
-					// wait until: join fetch adds results, cleanup runs, or the join-wait times out
-					await waitForUpdate();
-
-					// re-arm the deferred for the next cycle (only if joining is enabled and we’re not done)
-					if (updateDeferred && !doneFn()) {
-						updateDeferred = pDefer<void>();
-					}
+					await waitForUpdateAndResetDeferred();
 				}
 				cleanupAndDone();
 				return result;
@@ -2453,6 +2885,26 @@ export class DocumentIndex<
 				cleanupAndDone();
 				return batch[0];
 			},
+			[Symbol.asyncIterator]: async function* () {
+				drain = true;
+				let c = 0;
+				while (doneFn() !== true) {
+					const batch = await next(100);
+					c += batch.length;
+					if (c > WARNING_WHEN_ITERATING_FOR_MORE_THAN) {
+						logger.warn(
+							"Iterating for more than " +
+								WARNING_WHEN_ITERATING_FOR_MORE_THAN +
+								" results",
+						);
+					}
+					for (const entry of batch) {
+						yield entry;
+					}
+					await waitForUpdateAndResetDeferred();
+				}
+				cleanupAndDone();
+			},
 		};
 	}
 
@@ -2568,26 +3020,15 @@ export class DocumentIndex<
 	}
 
 	public async waitFor(
-		other:
-			| PublicSignKey
-			| PeerId
-			| string
-			| (PublicSignKey | string | PeerId)[],
-		options?: { signal?: AbortSignal; timeout?: number },
-	): Promise<void> {
-		await super.waitFor(other, options);
-		const ids = Array.isArray(other) ? other : [other];
-		const expectedHashes = new Set(
-			ids.map((x) =>
-				typeof x === "string"
-					? x
-					: x instanceof PublicSignKey
-						? x.hashcode()
-						: getPublicKeyFromPeerId(x).hashcode(),
-			),
-		);
-
-		for (const key of expectedHashes) {
+		other: PeerRefs,
+		options?: {
+			seek?: "any" | "present";
+			signal?: AbortSignal;
+			timeout?: number;
+		},
+	): Promise<string[]> {
+		const hashes = await super.waitFor(other, options);
+		for (const key of hashes) {
 			await waitFor(
 				async () =>
 					(await this._log.replicationIndex.count({ query: { hash: key } })) >
@@ -2595,5 +3036,6 @@ export class DocumentIndex<
 				options,
 			);
 		}
+		return hashes;
 	}
 }