@peerbit/document 9.13.10 → 10.0.0

package/dist/src/search.js

@@ -25,34 +25,65 @@ import pDefer, {} from "p-defer";
 import { concat, fromString } from "uint8arrays";
 import { copySerialization } from "./borsh.js";
 import { MAX_BATCH_SIZE } from "./constants.js";
-import MostCommonQueryPredictor from "./most-common-query-predictor.js";
+import MostCommonQueryPredictor, { idAgnosticQueryKey, } from "./most-common-query-predictor.js";
 import { isPutOperation } from "./operation.js";
 import { Prefetch } from "./prefetch.js";
 import { ResumableIterators } from "./resumable-iterator.js";
 const WARNING_WHEN_ITERATING_FOR_MORE_THAN = 1e5;
 const logger = loggerFn({ module: "document-index" });
-const coerceQuery = (query, options) => {
-    let replicate = typeof options?.remote !== "boolean" ? options?.remote?.replicate : false;
+const coerceQuery = (query, options, compatibility) => {
+    const replicate = typeof options?.remote !== "boolean" ? options?.remote?.replicate : false;
+    const shouldResolve = options?.resolve !== false;
+    const useLegacyRequests = compatibility != null && compatibility <= 9;
     if (query instanceof types.SearchRequestIndexed &&
         query.replicate === false &&
         replicate) {
         query.replicate = true;
         return query;
     }
-    if (query instanceof types.SearchRequest) {
+    if (query instanceof types.SearchRequest ||
+        query instanceof types.SearchRequestIndexed) {
+        return query;
+    }
+    if (query instanceof types.IterationRequest) {
+        if (useLegacyRequests) {
+            if (query.resolve === false) {
+                return new types.SearchRequestIndexed({
+                    query: query.query,
+                    sort: query.sort,
+                    fetch: query.fetch,
+                    replicate: query.replicate ?? replicate,
+                });
+            }
+            return new types.SearchRequest({
+                query: query.query,
+                sort: query.sort,
+                fetch: query.fetch,
+            });
+        }
         return query;
     }
     const queryObject = query;
-    return options?.resolve || options?.resolve == null
-        ? new types.SearchRequest({
-            query: indexerTypes.toQuery(queryObject.query),
-            sort: indexerTypes.toSort(query.sort),
-        })
-        : new types.SearchRequestIndexed({
+    if (useLegacyRequests) {
+        if (shouldResolve) {
+            return new types.SearchRequest({
+                query: indexerTypes.toQuery(queryObject.query),
+                sort: indexerTypes.toSort(queryObject.sort),
+            });
+        }
+        return new types.SearchRequestIndexed({
             query: indexerTypes.toQuery(queryObject.query),
-            sort: indexerTypes.toSort(query.sort),
+            sort: indexerTypes.toSort(queryObject.sort),
             replicate,
         });
+    }
+    return new types.IterationRequest({
+        query: indexerTypes.toQuery(queryObject.query),
+        sort: indexerTypes.toSort(queryObject.sort),
+        fetch: 10,
+        resolve: shouldResolve,
+        replicate: shouldResolve ? false : replicate,
+    });
 };
 const introduceEntries = async (queryRequest, responses, documentType, indexedType, sync, options) => {
     const results = [];
@@ -94,6 +125,30 @@ const dedup = (allResult, dedupBy) => {
     }
     return dedup;
 };
+const resolvesDocuments = (req) => {
+    if (!req) {
+        return true;
+    }
+    if (req instanceof types.SearchRequestIndexed) {
+        return false;
+    }
+    if (req instanceof types.IterationRequest) {
+        return req.resolve !== false;
+    }
+    return true;
+};
+const replicatesIndex = (req) => {
+    if (!req) {
+        return false;
+    }
+    if (req instanceof types.SearchRequestIndexed) {
+        return req.replicate === true;
+    }
+    if (req instanceof types.IterationRequest) {
+        return req.replicate === true;
+    }
+    return false;
+};
 function isSubclassOf(SubClass, SuperClass) {
     // Start with the immediate parent of SubClass
     let proto = Object.getPrototypeOf(SubClass);
@@ -106,6 +161,7 @@ function isSubclassOf(SubClass, SuperClass) {
     return false;
 }
 const DEFAULT_TIMEOUT = 1e4;
+const DISCOVER_TIMEOUT_FALLBACK = 500;
 const DEFAULT_INDEX_BY = "id";
 const isTransformerWithFunction = (options) => {
     return options.transform != null;
@@ -154,9 +210,11 @@ let DocumentIndex = class DocumentIndex extends Program {
     documentEvents;
     _joinListener;
     _resultQueue;
+    iteratorKeepAliveTimers;
     constructor(properties) {
         super();
         this._query = properties?.query || new RPC();
+        this.iteratorKeepAliveTimers = new Map();
     }
     get valueEncoding() {
         return this._valueEncoding;
@@ -317,7 +375,8 @@ let DocumentIndex = class DocumentIndex extends Program {
         }
         if (this.prefetch?.predictor &&
             (query instanceof types.SearchRequest ||
-                query instanceof types.SearchRequestIndexed)) {
+                query instanceof types.SearchRequestIndexed ||
+                query instanceof types.IterationRequest)) {
             const { ignore } = this.prefetch.predictor.onRequest(query, {
                 from: ctx.from,
             });
@@ -334,6 +393,7 @@ let DocumentIndex = class DocumentIndex extends Program {
     async handleSearchRequest(query, ctx) {
         if (this.canSearch &&
             (query instanceof types.SearchRequest ||
+                query instanceof types.IterationRequest ||
                 query instanceof types.CollectNextRequest) &&
             !(await this.canSearch(query, ctx.from))) {
             return new types.NoAccess();
@@ -342,11 +402,13 @@ let DocumentIndex = class DocumentIndex extends Program {
             this.processCloseIteratorRequest(query, ctx.from);
         }
         else {
-            const shouldIncludedIndexedResults = this.includeIndexed &&
-                (query instanceof types.SearchRequest ||
-                    (query instanceof types.CollectNextRequest &&
-                        this._resultQueue.get(query.idString)?.fromQuery instanceof
-                            types.SearchRequest)); // we do this check here because this._resultQueue might be emptied when this.processQuery is called
+            const fromQueued = query instanceof types.CollectNextRequest
+                ? this._resultQueue.get(query.idString)?.fromQuery
+                : undefined;
+            const queryResolvesDocuments = query instanceof types.CollectNextRequest
+                ? resolvesDocuments(fromQueued)
+                : resolvesDocuments(query);
+            const shouldIncludedIndexedResults = this.includeIndexed && queryResolvesDocuments;
             const results = await this.processQuery(query, ctx.from, false, {
                 canRead: this.canRead,
             });
@@ -585,22 +647,29 @@ let DocumentIndex = class DocumentIndex extends Program {
             };
         }
         let results;
+        const runAndClose = async (req) => {
+            const response = await this.queryCommence(req, coercedOptions);
+            this._resumableIterators.close({ idString: req.idString });
+            this.cancelIteratorKeepAlive(req.idString);
+            return response;
+        };
         const resolve = coercedOptions?.resolve || coercedOptions?.resolve == null;
         let requestClazz = resolve
             ? types.SearchRequest
             : types.SearchRequestIndexed;
         if (key instanceof Uint8Array) {
-            results = await this.queryCommence(new requestClazz({
+            const request = new requestClazz({
                 query: [
                     new indexerTypes.ByteMatchQuery({ key: this.indexBy, value: key }),
                 ],
-            }), coercedOptions);
+            });
+            results = await runAndClose(request);
         }
         else {
             const indexableKey = indexerTypes.toIdeable(key);
             if (typeof indexableKey === "number" ||
                 typeof indexableKey === "bigint") {
-                results = await this.queryCommence(new requestClazz({
+                const request = new requestClazz({
                     query: [
                         new indexerTypes.IntegerCompare({
                             key: this.indexBy,
@@ -608,27 +677,44 @@ let DocumentIndex = class DocumentIndex extends Program {
                             value: indexableKey,
                         }),
                     ],
-                }), coercedOptions);
+                });
+                results = await runAndClose(request);
             }
             else if (typeof indexableKey === "string") {
-                results = await this.queryCommence(new requestClazz({
+                const request = new requestClazz({
                     query: [
                         new indexerTypes.StringMatch({
                             key: this.indexBy,
                             value: indexableKey,
                         }),
                     ],
-                }), coercedOptions);
+                });
+                results = await runAndClose(request);
             }
             else if (indexableKey instanceof Uint8Array) {
-                results = await this.queryCommence(new requestClazz({
+                const request = new requestClazz({
                     query: [
                         new indexerTypes.ByteMatchQuery({
                             key: this.indexBy,
                             value: indexableKey,
                         }),
                     ],
-                }), coercedOptions);
+                });
+                results = await runAndClose(request);
+            }
+            else if (indexableKey instanceof ArrayBuffer) {
+                const request = new requestClazz({
+                    query: [
+                        new indexerTypes.ByteMatchQuery({
+                            key: this.indexBy,
+                            value: new Uint8Array(indexableKey),
+                        }),
+                    ],
+                });
+                results = await runAndClose(request);
+            }
+            else {
+                throw new Error("Unsupported key type");
             }
         }
         // if we are to resolve the document we need to go through all results and replace the results with the resolved values
@@ -711,23 +797,42 @@ let DocumentIndex = class DocumentIndex extends Program {
         }
         let indexedResult = undefined;
         let fromQuery;
+        let keepAliveRequest;
         if (query instanceof types.SearchRequest ||
-            query instanceof types.SearchRequestIndexed) {
+            query instanceof types.SearchRequestIndexed ||
+            query instanceof types.IterationRequest) {
             fromQuery = query;
-            indexedResult = await this._resumableIterators.iterateAndFetch(query);
+            if (!isLocal &&
+                query instanceof types.IterationRequest &&
+                query.keepAliveTtl != null) {
+                keepAliveRequest = query;
+            }
+            indexedResult = await this._resumableIterators.iterateAndFetch(query, {
+                keepAlive: keepAliveRequest !== undefined,
+            });
         }
         else if (query instanceof types.CollectNextRequest) {
-            fromQuery =
-                prevQueued?.fromQuery ||
-                    this._resumableIterators.queues.get(query.idString)?.request;
+            const cachedRequest = prevQueued?.fromQuery ||
+                this._resumableIterators.queues.get(query.idString)?.request;
+            fromQuery = cachedRequest;
+            if (!isLocal &&
+                cachedRequest instanceof types.IterationRequest &&
+                cachedRequest.keepAliveTtl != null) {
+                keepAliveRequest = cachedRequest;
+            }
             indexedResult =
                 prevQueued?.keptInIndex === 0
                     ? []
-                    : await this._resumableIterators.next(query);
+                    : await this._resumableIterators.next(query, {
+                        keepAlive: keepAliveRequest !== undefined,
+                    });
         }
         else {
             throw new Error("Unsupported");
         }
+        if (!isLocal && keepAliveRequest) {
+            this.scheduleIteratorKeepAlive(query.idString, keepAliveRequest.keepAliveTtl);
+        }
         let resultSize = 0;
         let toIterate = prevQueued
             ? [...prevQueued.queue, ...indexedResult]
@@ -751,6 +856,8 @@ let DocumentIndex = class DocumentIndex extends Program {
             this._resultQueue.set(query.idString, prevQueued);
         }
         const filteredResults = [];
+        const resolveDocumentsFlag = resolvesDocuments(fromQuery);
+        const replicateIndexFlag = replicatesIndex(fromQuery);
         for (const result of toIterate) {
             if (!isLocal) {
                 resultSize += result.value.__context.size;
@@ -764,7 +871,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                 !(await options.canRead(indexedUnwrapped, from))) {
                 continue;
             }
-            if (fromQuery instanceof types.SearchRequest) {
+            if (resolveDocumentsFlag) {
                 const value = await this.resolveDocument({
                     indexed: result.value,
                     head: result.value.__context.head,
@@ -779,11 +886,11 @@ let DocumentIndex = class DocumentIndex extends Program {
                     indexed: indexedUnwrapped,
                 }));
             }
-            else if (fromQuery instanceof types.SearchRequestIndexed) {
+            else {
                 const context = result.value.__context;
                 const head = await this._log.log.get(context.head);
                 // assume remote peer will start to replicate (TODO is this ideal?)
-                if (fromQuery.replicate) {
+                if (replicateIndexFlag) {
                     this._log.addPeersToGidPeerHistory(context.gid, [from.hashcode()]);
                 }
                 filteredResults.push(new types.ResultIndexedValue({
@@ -798,11 +905,53 @@ let DocumentIndex = class DocumentIndex extends Program {
             results: filteredResults,
             kept: BigInt(kept + (prevQueued?.queue.length || 0)),
         });
+        /* console.debug("[DocumentIndex] processQuery", {
+            id: query.idString,
+            isLocal,
+            batchLength: filteredResults.length,
+            kept: results.kept,
+            source: from.hashcode(),
+        }); */
         if (!isLocal && results.kept === 0n) {
             this.clearResultsQueue(query);
         }
         return results;
     }
+    scheduleIteratorKeepAlive(idString, ttl) {
+        if (ttl == null) {
+            return;
+        }
+        const ttlNumber = Number(ttl);
+        if (!Number.isFinite(ttlNumber) || ttlNumber <= 0) {
+            return;
+        }
+        // Cap max timeout to 1 day	(TODO make configurable?)
+        const delay = Math.max(1, Math.min(ttlNumber, 86400000));
+        this.cancelIteratorKeepAlive(idString);
+        const timers = this.iteratorKeepAliveTimers ??
+            (this.iteratorKeepAliveTimers = new Map());
+        const timer = setTimeout(() => {
+            timers.delete(idString);
+            const queued = this._resultQueue.get(idString);
+            if (queued) {
+                clearTimeout(queued.timeout);
+                this._resultQueue.delete(idString);
+            }
+            this._resumableIterators.close({ idString });
+        }, delay);
+        timers.set(idString, timer);
+    }
+    cancelIteratorKeepAlive(idString) {
+        const timers = this.iteratorKeepAliveTimers;
+        if (!timers) {
+            return;
+        }
+        const timer = timers.get(idString);
+        if (timer) {
+            clearTimeout(timer);
+            timers.delete(idString);
+        }
+    }
     clearResultsQueue(query) {
         const queue = this._resultQueue.get(query.idString);
         if (queue) {
@@ -817,6 +966,7 @@ let DocumentIndex = class DocumentIndex extends Program {
         for (const [key, queue] of this._resultQueue) {
             clearTimeout(queue.timeout);
             this._resultQueue.delete(key);
+            this.cancelIteratorKeepAlive(key);
             this._resumableIterators.close({ idString: key });
         }
     }
@@ -950,6 +1100,7 @@ let DocumentIndex = class DocumentIndex extends Program {
             logger.info("Ignoring close iterator request from different peer");
             return;
         }
+        this.cancelIteratorKeepAlive(query.idString);
         this.clearResultsQueue(query);
         return this._resumableIterators.close(query);
     }
@@ -1126,7 +1277,7 @@ let DocumentIndex = class DocumentIndex extends Program {
      */
     async search(queryRequest, options) {
         // Set fetch to search size, or max value (default to max u32 (4294967295))
-        const coercedRequest = coerceQuery(queryRequest, options);
+        const coercedRequest = coerceQuery(queryRequest, options, this.compatibility);
         coercedRequest.fetch = coercedRequest.fetch ?? 0xffffffff;
         // So that the iterator is pre-fetching the right amount of entries
         const iterator = this.iterate(coercedRequest, options);
@@ -1168,19 +1319,42 @@ let DocumentIndex = class DocumentIndex extends Program {
     /**
      * Query and retrieve documents in a iterator
      * @param queryRequest
-     * @param options
+     * @param optionsArg
      * @returns
      */
-    iterate(queryRequest, options) {
+    iterate(queryRequest, optionsArg) {
+        let options = optionsArg;
         if (queryRequest instanceof types.SearchRequest &&
             options?.resolve === false) {
             throw new Error("Cannot use resolve=false with SearchRequest"); // TODO make this work
         }
-        let queryRequestCoerced = coerceQuery(queryRequest ?? {}, options);
+        let queryRequestCoerced = coerceQuery(queryRequest ?? {}, options, this.compatibility);
+        const { mergePolicy, push: pushUpdates, callbacks: updateCallbacksRaw, } = normalizeUpdatesOption(options?.updates);
+        const hasLiveUpdates = mergePolicy !== undefined;
+        const originalRemote = options?.remote;
+        let remoteOptions = typeof originalRemote === "boolean"
+            ? originalRemote
+            : originalRemote
+                ? { ...originalRemote }
+                : undefined;
+        if (pushUpdates && remoteOptions !== false) {
+            if (typeof remoteOptions === "object") {
+                if (remoteOptions.replicate !== true) {
+                    remoteOptions.replicate = true;
+                }
+            }
+            else if (remoteOptions === undefined || remoteOptions === true) {
+                remoteOptions = { replicate: true };
+            }
+        }
+        if (remoteOptions !== originalRemote) {
+            options = Object.assign({}, options, { remote: remoteOptions });
+        }
         let resolve = options?.resolve !== false;
-        if (options?.remote &&
+        if (!(queryRequestCoerced instanceof types.IterationRequest) &&
+            options?.remote &&
             typeof options.remote !== "boolean" &&
-            options.remote.replicate &&
+            (options.remote.replicate || pushUpdates) &&
             options?.resolve !== false) {
             if ((queryRequest instanceof types.SearchRequestIndexed === false &&
                 this.compatibility == null) ||
@@ -1195,7 +1369,7 @@ let DocumentIndex = class DocumentIndex extends Program {
         }
         let replicate = options?.remote &&
             typeof options.remote !== "boolean" &&
-            options.remote.replicate;
+            (options.remote.replicate || pushUpdates);
         if (replicate &&
             queryRequestCoerced instanceof types.SearchRequestIndexed) {
             queryRequestCoerced.replicate = true;
@@ -1237,6 +1411,7 @@ let DocumentIndex = class DocumentIndex extends Program {
             this.clearResultsQueue(queryRequestCoerced);
         };
         let warmupPromise = undefined;
+        let discoveredTargetHashes;
         if (typeof options?.remote === "object") {
             let waitForTime = undefined;
             if (options.remote.wait) {
@@ -1271,11 +1446,25 @@ let DocumentIndex = class DocumentIndex extends Program {
                 };
             }
             if (options.remote.reach?.discover) {
-                warmupPromise = this.waitFor(options.remote.reach.discover, {
+                const discoverTimeout = waitForTime ??
+                    (options.remote.wait ? DEFAULT_TIMEOUT : DISCOVER_TIMEOUT_FALLBACK);
+                const discoverPromise = this.waitFor(options.remote.reach.discover, {
                     signal: ensureController().signal,
                     seek: "present",
-                    timeout: waitForTime ?? DEFAULT_TIMEOUT,
+                    timeout: discoverTimeout,
+                })
+                    .then((hashes) => {
+                    discoveredTargetHashes = hashes;
+                })
+                    .catch((error) => {
+                    if (error instanceof TimeoutError || error instanceof AbortError) {
+                        discoveredTargetHashes = [];
+                        return;
+                    }
+                    throw error;
                 });
+                const prior = warmupPromise ?? Promise.resolve();
+                warmupPromise = prior.then(() => discoverPromise);
                 options.remote.reach.eager = true; // include the results from the discovered peer even if it is not mature
             }
             const waitPolicy = typeof options.remote.wait === "object"
@@ -1299,15 +1488,24 @@ let DocumentIndex = class DocumentIndex extends Program {
         const fetchFirst = async (n, fetchOptions) => {
             await warmupPromise;
             let hasMore = false;
+            const discoverTargets = typeof options?.remote === "object"
+                ? options.remote.reach?.discover
+                : undefined;
+            const initialRemoteTargets = discoveredTargetHashes !== undefined
+                ? discoveredTargetHashes
+                : discoverTargets?.map((pk) => pk.hashcode().toString());
+            const skipRemoteDueToDiscovery = typeof options?.remote === "object" &&
+                options.remote.reach?.discover &&
+                discoveredTargetHashes?.length === 0;
             queryRequestCoerced.fetch = n;
             await this.queryCommence(queryRequestCoerced, {
                 local: fetchOptions?.from != null ? false : options?.local,
-                remote: options?.remote !== false
+                remote: options?.remote !== false && !skipRemoteDueToDiscovery
                     ? {
                         ...(typeof options?.remote === "object"
                             ? options.remote
                             : {}),
-                        from: fetchOptions?.from,
+                        from: fetchOptions?.from ?? initialRemoteTargets,
                     }
                     : false,
                 resolve,
@@ -1322,13 +1520,20 @@ let DocumentIndex = class DocumentIndex extends Program {
                     }
                     else if (response instanceof types.Results) {
                         const results = response;
+                        const existingBuffer = peerBufferMap.get(from.hashcode());
+                        const buffer = existingBuffer?.buffer || [];
                         if (results.kept === 0n && results.results.length === 0) {
+                            if (keepRemoteAlive) {
+                                peerBufferMap.set(from.hashcode(), {
+                                    buffer,
+                                    kept: Number(response.kept),
+                                });
+                            }
                             return;
                         }
                         if (results.kept > 0n) {
                             hasMore = true;
                         }
-                        const buffer = peerBufferMap.get(from.hashcode())?.buffer || [];
                         for (const result of results.results) {
                             const indexKey = indexerTypes.toId(this.indexByResolver(result.value)).primitive;
                             if (result instanceof types.ResultValue) {
@@ -1380,6 +1585,15 @@ let DocumentIndex = class DocumentIndex extends Program {
                     }
                 },
             }, fetchOptions?.fetchedFirstForRemote);
+            /* console.debug(
+                "[DocumentIndex] fetchFirst",
+                {
+                    id: queryRequestCoerced.idString,
+                    requestedFrom: fetchOptions?.from,
+                    initialRemoteTargets,
+                    keepRemoteAlive,
+                },
+            ); */
             if (!hasMore) {
                 maybeSetDone();
             }
@@ -1403,7 +1617,8 @@ let DocumentIndex = class DocumentIndex extends Program {
             let resultsLeft = 0;
             for (const [peer, buffer] of peerBufferMap) {
                 if (buffer.buffer.length < n) {
-                    if (buffer.kept === 0) {
+                    const hasExistingRemoteResults = buffer.kept > 0;
+                    if (!hasExistingRemoteResults && !keepRemoteAlive) {
                         if (peerBufferMap.get(peer)?.buffer.length === 0) {
                             peerBufferMap.delete(peer); // No more results
                         }
@@ -1411,9 +1626,21 @@ let DocumentIndex = class DocumentIndex extends Program {
                     }
                     // TODO buffer more than deleted?
                     // TODO batch to multiple 'to's
+                    const lacking = n - buffer.buffer.length;
+                    const amount = lacking > 0 ? lacking : keepRemoteAlive ? 1 : 0;
+                    /* console.debug("[DocumentIndex] fetchAtLeast loop", {
+                        peer,
+                        bufferLength: buffer.buffer.length,
+                        bufferKept: buffer.kept,
+                        amount,
+                        keepRemoteAlive,
+                    }); */
+                    if (amount <= 0) {
+                        continue;
+                    }
                     const collectRequest = new types.CollectNextRequest({
                         id: queryRequestCoerced.id,
-                        amount: n - buffer.buffer.length,
+                        amount,
                     });
                     // Fetch locally?
                     if (peer === this.node.identity.publicKey.hashcode()) {
@@ -1424,7 +1651,8 @@ let DocumentIndex = class DocumentIndex extends Program {
                             .then(async (results) => {
                             resultsLeft += Number(results.kept);
                             if (results.results.length === 0) {
-                                if (peerBufferMap.get(peer)?.buffer.length === 0) {
+                                if (!keepRemoteAlive &&
+                                    peerBufferMap.get(peer)?.buffer.length === 0) {
                                     peerBufferMap.delete(peer); // No more results
                                 }
                             }
@@ -1516,7 +1744,8 @@ let DocumentIndex = class DocumentIndex extends Program {
                                     return;
                                 }
                                 if (response.response.results.length === 0) {
-                                    if (peerBufferMap.get(peer)?.buffer.length === 0) {
+                                    if (!keepRemoteAlive &&
+                                        peerBufferMap.get(peer)?.buffer.length === 0) {
                                         peerBufferMap.delete(peer); // No more results
                                     }
                                 }
@@ -1684,32 +1913,75 @@ let DocumentIndex = class DocumentIndex extends Program {
         let joinListener;
         let fetchedFirstForRemote = undefined;
         let updateDeferred;
-        const signalUpdate = () => updateDeferred?.resolve();
+        const signalUpdate = (reason) => {
+            if (reason) {
+                /* console.debug("[DocumentIndex] signalUpdate", {
+                    id: queryRequestCoerced.idString,
+                    reason,
+                }); */
+            }
+            updateDeferred?.resolve();
+        };
         const _waitForUpdate = () => updateDeferred ? updateDeferred.promise : Promise.resolve();
         // ---------------- Live updates wiring (sorted-only with optional filter) ----------------
-        const normalizeUpdatesOption = (u) => {
-            if (u == null || u === false)
-                return undefined;
-            if (u === true)
+        function normalizeUpdatesOption(u) {
+            const identityFilter = (evt) => evt;
+            const buildMergePolicy = (merge, defaultEnabled) => {
+                const effective = merge === undefined ? (defaultEnabled ? true : undefined) : merge;
+                if (effective === undefined || effective === false) {
+                    return undefined;
+                }
+                if (effective === true) {
+                    return {
+                        merge: {
+                            filter: identityFilter,
+                        },
+                    };
+                }
                 return {
                     merge: {
-                        filter: (evt) => evt,
+                        filter: effective.filter ?? identityFilter,
                     },
                 };
+            };
+            if (u == null || u === false) {
+                return { push: false };
+            }
+            if (u === true) {
+                return {
+                    mergePolicy: buildMergePolicy(true, true),
+                    push: false,
+                };
+            }
+            if (typeof u === "string") {
+                if (u === "remote") {
+                    return { push: true };
+                }
+                if (u === "local") {
+                    return {
+                        mergePolicy: buildMergePolicy(true, true),
+                        push: false,
+                    };
+                }
+                if (u === "all") {
+                    return {
+                        mergePolicy: buildMergePolicy(true, true),
+                        push: true,
+                    };
+                }
+            }
             if (typeof u === "object") {
+                const hasMergeProp = Object.prototype.hasOwnProperty.call(u, "merge");
+                const mergeValue = hasMergeProp ? u.merge : undefined;
                 return {
-                    merge: u.merge
-                        ? {
-                            filter: typeof u.merge === "object" ? u.merge.filter : (evt) => evt,
-                        }
-                        : {},
+                    mergePolicy: buildMergePolicy(mergeValue, !hasMergeProp || mergeValue === undefined),
+                    push: Boolean(u.push),
+                    callbacks: u,
                 };
             }
-            return undefined;
-        };
-        const updateCallbacks = typeof options?.updates === "object" ? options.updates : undefined;
-        const mergePolicy = normalizeUpdatesOption(options?.updates);
-        const hasLiveUpdates = mergePolicy !== undefined;
+            return { push: false };
+        }
+        const updateCallbacks = updateCallbacksRaw;
         let pendingResultsReason;
         let hasDeliveredResults = false;
         const emitOnResults = async (batch, defaultReason) => {
@@ -1735,6 +2007,130 @@ let DocumentIndex = class DocumentIndex extends Program {
         if (hasLiveUpdates && !updateDeferred) {
             updateDeferred = pDefer();
         }
+        const keepRemoteAlive = (options?.closePolicy === "manual" || hasLiveUpdates || pushUpdates) &&
+            remoteOptions !== false;
+        if (queryRequestCoerced instanceof types.IterationRequest) {
+            queryRequestCoerced.resolve = resolve;
+            queryRequestCoerced.fetch = queryRequestCoerced.fetch ?? 10;
+            const replicateFlag = !resolve && replicate ? true : false;
+            queryRequestCoerced.replicate = replicateFlag;
+            const ttlSource = typeof remoteOptions === "object" &&
+                typeof remoteOptions?.wait === "object"
+                ? (remoteOptions.wait.timeout ?? DEFAULT_TIMEOUT)
+                : DEFAULT_TIMEOUT;
+            queryRequestCoerced.keepAliveTtl = keepRemoteAlive
+                ? BigInt(ttlSource)
+                : undefined;
+            queryRequestCoerced.pushUpdates = pushUpdates ? true : undefined;
+            queryRequestCoerced.mergeUpdates = mergePolicy?.merge ? true : undefined;
+        }
+        if (pushUpdates && this.prefetch?.accumulator) {
+            const targetPrefetchKey = idAgnosticQueryKey(queryRequestCoerced);
+            const mergePrefetchedResults = async (from, results) => {
+                const peerHash = from.hashcode();
+                const existingBuffer = peerBufferMap.get(peerHash);
+                const buffer = existingBuffer?.buffer || [];
+                if (results.kept === 0n && results.results.length === 0) {
+                    peerBufferMap.set(peerHash, {
+                        buffer,
+                        kept: Number(results.kept),
+                    });
+                    return;
+                }
+                for (const result of results.results) {
+                    const indexKey = indexerTypes.toId(this.indexByResolver(result.value)).primitive;
+                    if (result instanceof types.ResultValue) {
+                        const existingIndexed = indexedPlaceholders?.get(indexKey);
+                        if (existingIndexed) {
+                            existingIndexed.value =
+                                result.value;
+                            existingIndexed.context = result.context;
+                            existingIndexed.from = from;
+                            existingIndexed.indexed = await this.resolveIndexed(result, results.results);
+                            indexedPlaceholders?.delete(indexKey);
+                            continue;
+                        }
+                        if (visited.has(indexKey)) {
+                            continue;
+                        }
+                        visited.add(indexKey);
+                        buffer.push({
+                            value: result.value,
+                            context: result.context,
+                            from,
+                            indexed: await this.resolveIndexed(result, results.results),
+                        });
+                    }
+                    else {
+                        if (visited.has(indexKey) && !indexedPlaceholders?.has(indexKey)) {
+                            continue;
+                        }
+                        visited.add(indexKey);
+                        const indexed = coerceWithContext(result.indexed || result.value, result.context);
+                        const placeholder = {
+                            value: result.value,
+                            context: result.context,
+                            from,
+                            indexed,
+                        };
+                        buffer.push(placeholder);
+                        ensureIndexedPlaceholders().set(indexKey, placeholder);
+                    }
+                }
+                peerBufferMap.set(peerHash, {
+                    buffer,
+                    kept: Number(results.kept),
+                });
+            };
+            const consumePrefetch = async (consumable) => {
+                const request = consumable.response?.request;
+                if (!request) {
+                    return;
+                }
+                if (idAgnosticQueryKey(request) !== targetPrefetchKey) {
+                    return;
+                }
+                /* console.debug("[DocumentIndex] prefetch match", {
+                    iterator: queryRequestCoerced.idString,
+                    source: consumable.from?.hashcode(),
+                });
+ */
+                try {
+                    const prepared = await introduceEntries(queryRequestCoerced, [
+                        {
+                            response: consumable.response.results,
+                            from: consumable.from,
+                        },
+                    ], this.documentType, this.indexedType, this._sync, options);
+                    for (const response of prepared) {
+                        if (!response.from) {
+                            continue;
+                        }
+                        const payload = response.response;
+                        if (!(payload instanceof types.Results)) {
+                            continue;
+                        }
+                        await mergePrefetchedResults(response.from, payload);
+                    }
+                    if (!pendingResultsReason) {
+                        pendingResultsReason = "change";
+                    }
+                    signalUpdate("prefetch-add");
+                }
+                catch (error) {
+                    logger.warn("Failed to merge prefetched results", error);
+                }
+            };
+            const onPrefetchAdd = (evt) => {
+                void consumePrefetch(evt.detail.consumable);
+            };
+            this.prefetch.accumulator.addEventListener("add", onPrefetchAdd);
+            const cleanupDefault = cleanup;
+            cleanup = () => {
+                this.prefetch?.accumulator.removeEventListener("add", onPrefetchAdd);
+                return cleanupDefault();
+            };
+        }
         let updatesCleanup;
         if (hasLiveUpdates) {
             const localHash = this.node.identity.publicKey.hashcode();
@@ -1767,6 +2163,11 @@ let DocumentIndex = class DocumentIndex extends Program {
                 return value;
             };
             const onChange = async (evt) => {
+                /* console.debug("[DocumentIndex] onChange event", {
+                    id: queryRequestCoerced.idString,
+                    added: evt.detail.added?.length,
+                    removed: evt.detail.removed?.length,
+                }); */
                 // Optional filter to mutate/suppress change events
                 let filtered = evt.detail;
                 if (mergePolicy?.merge?.filter) {
@@ -1862,6 +2263,12 @@ let DocumentIndex = class DocumentIndex extends Program {
                                 indexed: indexedCandidate,
                             };
                             buf.buffer.push(placeholder);
+                            /* console.debug("[DocumentIndex] buffered change", {
+                                id: queryRequestCoerced.idString,
+                                placeholderId: (valueForBuffer as any)?.id,
+                                peer: localHash,
+                                bufferSize: buf.buffer.length,
+                            }); */
                             if (!resolve) {
                                 ensureIndexedPlaceholders().set(id, placeholder);
                             }
@@ -1875,7 +2282,13 @@ let DocumentIndex = class DocumentIndex extends Program {
                         }
                         if (changeForCallback.added.length > 0 ||
                             changeForCallback.removed.length > 0) {
+                            /* console.debug("[DocumentIndex] changeForCallback", {
+                                id: queryRequestCoerced.idString,
+                                added: changeForCallback.added.map((x) => (x as any)?.id),
+                                removed: changeForCallback.removed.map((x) => (x as any)?.id),
+                            }); */
                             updateCallbacks?.onChange?.(changeForCallback);
+                            signalUpdate("change");
                         }
                     }
                 }
@@ -1962,8 +2375,8 @@ let DocumentIndex = class DocumentIndex extends Program {
                 return cleanupDefault();
             };
         }
-        if (options?.closePolicy === "manual") {
-            let prevMaybeSetDone = maybeSetDone;
+        if (keepRemoteAlive) {
+            const prevMaybeSetDone = maybeSetDone;
             maybeSetDone = () => {
                 if (drain) {
                     prevMaybeSetDone();
@@ -1985,7 +2398,16 @@ let DocumentIndex = class DocumentIndex extends Program {
             close,
             next,
             done: doneFn,
-            pending: () => {
+            pending: async () => {
+                try {
+                    await fetchPromise;
+                    if (!done && keepRemoteAlive) {
+                        await fetchAtLeast(1);
+                    }
+                }
+                catch (error) {
+                    logger.warn("Failed to refresh iterator pending state", error);
+                }
                 let pendingCount = 0;
                 for (const buffer of peerBufferMap.values()) {
                     pendingCount += buffer.kept + buffer.buffer.length;