@peerbit/document 9.12.17 → 9.13.0-5da22a7

package/dist/src/search.js

@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 import { field, serialize, variant } from "@dao-xyz/borsh";
 import { Cache } from "@peerbit/cache";
-import { PublicSignKey, getPublicKeyFromPeerId, sha256Base64Sync, } from "@peerbit/crypto";
+import { PublicSignKey, sha256Base64Sync, } from "@peerbit/crypto";
 import * as types from "@peerbit/document-interface";
 import { CachedIndex } from "@peerbit/indexer-cache";
 import * as indexerTypes from "@peerbit/indexer-interface";
@@ -19,8 +19,8 @@ import { logger as loggerFn } from "@peerbit/logger";
 import { ClosedError, Program } from "@peerbit/program";
 import { MissingResponsesError, RPC, queryAll, } from "@peerbit/rpc";
 import { SharedLog, } from "@peerbit/shared-log";
-import { DataMessage, SilentDelivery } from "@peerbit/stream-interface";
-import { AbortError, waitFor } from "@peerbit/time";
+import { DataMessage, SilentDelivery, } from "@peerbit/stream-interface";
+import { AbortError, TimeoutError, waitFor } from "@peerbit/time";
 import pDefer, {} from "p-defer";
 import { concat, fromString } from "uint8arrays";
 import { copySerialization } from "./borsh.js";
@@ -29,6 +29,7 @@ import MostCommonQueryPredictor from "./most-common-query-predictor.js";
 import { isPutOperation } from "./operation.js";
 import { Prefetch } from "./prefetch.js";
 import { ResumableIterators } from "./resumable-iterator.js";
+const WARNING_WHEN_ITERATING_FOR_MORE_THAN = 1e5;
 const logger = loggerFn({ module: "document-index" });
 const coerceQuery = (query, options) => {
     let replicate = typeof options?.remote !== "boolean" ? options?.remote?.replicate : false;
@@ -464,14 +465,13 @@ let DocumentIndex = class DocumentIndex extends Program {
                     ? { ...options.remote }
                     : {};
             if (baseRemote) {
-                const prevJoining = baseRemote.joining;
-                if (!prevJoining ||
-                    prevJoining === true ||
-                    (typeof prevJoining === "object" &&
-                        (prevJoining.waitFor || 0) < options.waitFor)) {
-                    baseRemote.joining = {
-                        ...(typeof prevJoining === "object" ? prevJoining : {}),
-                        waitFor: options.waitFor,
+                const waitPolicy = baseRemote.wait;
+                if (!waitPolicy ||
+                    (typeof waitPolicy === "object" &&
+                        (waitPolicy.timeout || 0) < options.waitFor)) {
+                    baseRemote.wait = {
+                        ...(typeof waitPolicy === "object" ? waitPolicy : {}),
+                        timeout: options.waitFor,
                     };
                 }
             }
@@ -479,6 +479,7 @@ let DocumentIndex = class DocumentIndex extends Program {
             let joinListener;
             if (baseRemote) {
                 joinListener = this.attachJoinListener({
+                    eager: baseRemote.reach?.eager,
                     onPeer: async (pk) => {
                         if (cleanedUp)
                             return;
@@ -819,6 +820,98 @@ let DocumentIndex = class DocumentIndex extends Program {
             this._resumableIterators.close({ idString: key });
         }
     }
+    async waitForCoverReady(params) {
+        const { domain, eager, settle, timeout, signal, onTimeout = "proceed", } = params;
+        if (settle !== "any") {
+            return;
+        }
+        const properties = domain && "range" in domain
+            ? { range: domain.range }
+            : { args: domain?.args };
+        const selfHash = this.node.identity.publicKey.hashcode();
+        const ready = async () => {
+            const cover = await this._log.getCover(properties, { eager });
+            return cover.some((hash) => hash !== selfHash);
+        };
+        if (await ready()) {
+            return;
+        }
+        const deferred = pDefer();
+        let settled = false;
+        let cleaned = false;
+        let timer;
+        let checking = false;
+        const cleanup = () => {
+            if (cleaned) {
+                return;
+            }
+            cleaned = true;
+            this._log.events.removeEventListener("replicator:join", onEvent);
+            this._log.events.removeEventListener("replication:change", onEvent);
+            this._log.events.removeEventListener("replicator:mature", onEvent);
+            signal?.removeEventListener("abort", onAbort);
+            if (timer != null) {
+                clearTimeout(timer);
+                timer = undefined;
+            }
+        };
+        const resolve = () => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            cleanup();
+            deferred.resolve();
+        };
+        const reject = (error) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            cleanup();
+            deferred.reject(error);
+        };
+        const onAbort = () => reject(new AbortError());
+        const onEvent = async () => {
+            if (checking) {
+                return;
+            }
+            checking = true;
+            try {
+                if (await ready()) {
+                    resolve();
+                }
+            }
+            catch (error) {
+                reject(error instanceof Error ? error : new Error(String(error)));
+            }
+            finally {
+                checking = false;
+            }
+        };
+        if (signal) {
+            signal.addEventListener("abort", onAbort);
+        }
+        if (timeout > 0) {
+            timer = setTimeout(() => {
+                if (onTimeout === "error") {
+                    reject(new TimeoutError("Timeout waiting for participating replicator"));
+                }
+                else {
+                    resolve();
+                }
+            }, timeout);
+        }
+        this._log.events.addEventListener("replicator:join", onEvent);
+        this._log.events.addEventListener("replication:change", onEvent);
+        this._log.events.addEventListener("replicator:mature", onEvent);
+        try {
+            await deferred.promise;
+        }
+        finally {
+            cleanup();
+        }
+    }
     // Utility: attach a join listener that waits until a peer is a replicator,
     // then invokes the provided callback. Returns a detach function.
     attachJoinListener(params) {
@@ -837,6 +930,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                 await this._log
                     .waitForReplicator(pk, {
                     signal: params.signal,
+                    eager: params.eager,
                 })
                     .catch(() => undefined);
                 if (params.signal?.aborted)
@@ -865,7 +959,7 @@ let DocumentIndex = class DocumentIndex extends Program {
      * @param options
      * @returns
      */
-    async queryCommence(queryRequest, options) {
+    async queryCommence(queryRequest, options, fetchFirstForRemote) {
         const local = typeof options?.local === "boolean" ? options?.local : true;
         let remote = undefined;
         if (typeof options?.remote === "boolean") {
@@ -907,8 +1001,8 @@ let DocumentIndex = class DocumentIndex extends Program {
                 ? options?.remote?.from
                 : await this._log.getCover(remote.domain ?? { args: undefined }, {
                     roleAge: remote.minAge,
-                    eager: remote.eager,
-                    reachableOnly: !!remote.joining, // when we want to merge joining we can ignore pending to be online peers and instead consider them once they become online
+                    eager: remote.reach?.eager,
+                    reachableOnly: !!remote.wait, // when we want to merge joining we can ignore pending to be online peers and instead consider them once they become online
                 });
             if (replicatorGroups) {
                 const responseHandler = async (results) => {
@@ -923,6 +1017,11 @@ let DocumentIndex = class DocumentIndex extends Program {
                     if (hash === this.node.identity.publicKey.hashcode()) {
                         return false;
                     }
+                    if (fetchFirstForRemote?.has(hash)) {
+                        // we already fetched this one for remote, no need to do it again
+                        return false;
+                    }
+                    fetchFirstForRemote?.add(hash);
                     const resultAlready = this._prefetch?.accumulator.consume(queryRequest, hash);
                     if (resultAlready) {
                         (extraPromises || (extraPromises = [])).push((async () => {
@@ -1105,6 +1204,7 @@ let DocumentIndex = class DocumentIndex extends Program {
         const peerBufferMap = new Map();
         const visited = new Set();
         let done = false;
+        let drain = false; // if true, close on empty once (overrides manual)
         let first = false;
         // TODO handle join/leave while iterating
         const controller = new AbortController();
@@ -1113,60 +1213,88 @@ let DocumentIndex = class DocumentIndex extends Program {
         const peerBuffers = () => {
             return [...peerBufferMap.values()].map((x) => x.buffer).flat();
         };
-        let maybeSetDone;
-        let unsetDone;
-        let cleanup = () => { };
-        if (typeof options?.remote === "object" && options.remote.joining) {
-            let t0 = +new Date();
-            let waitForTime = typeof options.remote.joining === "boolean"
-                ? DEFAULT_TIMEOUT
-                : (options.remote.joining.waitFor ?? DEFAULT_TIMEOUT);
-            let setDoneIfTimeout = false;
-            maybeSetDone = () => {
-                if (t0 + waitForTime < +new Date()) {
-                    cleanup();
-                    done = true;
-                }
-                else {
-                    setDoneIfTimeout = true;
-                }
-            };
-            unsetDone = () => {
-                setDoneIfTimeout = false;
-                done = false;
-            };
-            let timeout = setTimeout(() => {
-                if (setDoneIfTimeout) {
-                    cleanup();
-                    done = true;
-                }
-            }, waitForTime);
-            cleanup = () => {
-                this.clearResultsQueue(queryRequestCoerced);
-                clearTimeout(timeout);
-            };
-        }
-        else {
-            maybeSetDone = () => {
-                cleanup();
-                done = true;
-            };
-            unsetDone = () => {
-                cleanup();
-                done = false;
-            };
-            cleanup = () => {
-                this.clearResultsQueue(queryRequestCoerced);
-            };
+        let maybeSetDone = () => {
+            cleanup();
+            done = true;
+        };
+        let unsetDone = () => {
+            cleanup();
+            done = false;
+        };
+        let cleanup = () => {
+            this.clearResultsQueue(queryRequestCoerced);
+        };
+        let warmupPromise = undefined;
+        if (typeof options?.remote === "object") {
+            let waitForTime = undefined;
+            if (options.remote.wait) {
+                let t0 = +new Date();
+                waitForTime =
+                    typeof options.remote.wait === "boolean"
+                        ? DEFAULT_TIMEOUT
+                        : (options.remote.wait.timeout ?? DEFAULT_TIMEOUT);
+                let setDoneIfTimeout = false;
+                maybeSetDone = () => {
+                    if (t0 + waitForTime < +new Date()) {
+                        cleanup();
+                        done = true;
+                    }
+                    else {
+                        setDoneIfTimeout = true;
+                    }
+                };
+                unsetDone = () => {
+                    setDoneIfTimeout = false;
+                    done = false;
+                };
+                let timeout = setTimeout(() => {
+                    if (setDoneIfTimeout) {
+                        cleanup();
+                        done = true;
+                    }
+                }, waitForTime);
+                cleanup = () => {
+                    this.clearResultsQueue(queryRequestCoerced);
+                    clearTimeout(timeout);
+                };
+            }
+            if (options.remote.reach?.discover) {
+                warmupPromise = this.waitFor(options.remote.reach.discover, {
+                    signal: controller.signal,
+                    seek: "present",
+                    timeout: waitForTime ?? DEFAULT_TIMEOUT,
+                });
+                options.remote.reach.eager = true; // include the results from the discovered peer even if it is not mature
+            }
+            const waitPolicy = typeof options.remote.wait === "object"
+                ? options.remote.wait
+                : undefined;
+            if (waitPolicy?.behavior === "block" &&
+                (waitPolicy.until ?? "any") === "any") {
+                const blockPromise = this.waitForCoverReady({
+                    domain: options.remote.domain,
+                    eager: options.remote.reach?.eager,
+                    settle: "any",
+                    timeout: waitPolicy.timeout ?? DEFAULT_TIMEOUT,
+                    signal: controller.signal,
+                    onTimeout: waitPolicy.onTimeout,
+                });
+                warmupPromise = warmupPromise
+                    ? Promise.all([warmupPromise, blockPromise]).then(() => undefined)
+                    : blockPromise;
+            }
         }
         const fetchFirst = async (n, fetchOptions) => {
+            await warmupPromise;
             let hasMore = false;
             queryRequestCoerced.fetch = n;
             await this.queryCommence(queryRequestCoerced, {
                 local: fetchOptions?.from != null ? false : options?.local,
                 remote: options?.remote !== false
                     ? {
-                        ...(typeof options?.remote === "object" ? options.remote : {}),
+                        ...(typeof options?.remote === "object"
+                            ? options.remote
+                            : {}),
                         from: fetchOptions?.from,
                     }
                     : false,
@@ -1226,7 +1354,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                         throw new Error("Unsupported result type: " + response?.constructor?.name);
                     }
                 },
-            });
+            }, fetchOptions?.fetchedFirstForRemote);
             if (!hasMore) {
                 maybeSetDone();
             }
@@ -1263,7 +1391,8 @@ let DocumentIndex = class DocumentIndex extends Program {
                         amount: n - buffer.buffer.length,
                     });
                     // Fetch locally?
-                    if (peer === this.node.identity.publicKey.hashcode()) {
+                    if (peer === this.node.identity.publicKey.hashcode() &&
+                        this._resumableIterators.has(queryRequestCoerced.idString)) {
                         promises.push(this.processQuery(collectRequest, this.node.identity.publicKey, true)
                             .then(async (results) => {
                             resultsLeft += Number(results.kept);
@@ -1398,9 +1527,10 @@ let DocumentIndex = class DocumentIndex extends Program {
             // get n next top entries, shift and pull more results
             const peerBuffersArr = peerBuffers();
             const results = peerBuffersArr.sort((a, b) => indexerTypes.extractSortCompare(a.indexed, b.indexed, queryRequestCoerced.sort));
-            const pendingMoreResults = n < results.length;
             lastValueInOrder = results[0] || lastValueInOrder;
+            const pendingMoreResults = n < results.length; // check if there are more results to fetch, before splicing
             const batch = results.splice(0, n);
+            const hasMore = !fetchedAll || pendingMoreResults;
             for (const result of batch) {
                 const arr = peerBufferMap.get(result.from.hashcode());
                 if (!arr) {
@@ -1412,7 +1542,6 @@ let DocumentIndex = class DocumentIndex extends Program {
                     arr.buffer.splice(idx, 1);
                 }
             }
-            const hasMore = !fetchedAll || pendingMoreResults;
             if (hasMore) {
                 unsetDone();
             }
@@ -1435,6 +1564,7 @@ let DocumentIndex = class DocumentIndex extends Program {
             else {
                 coercedBatch = batch.map((x) => coerceWithContext(coerceWithIndexed(x.value, x.indexed), x.context));
             }
+            // no extra queued-first/last in simplified API
             return dedup(coercedBatch, this.indexByResolver);
         };
         let cleanupAndDone = () => {
@@ -1471,19 +1601,107 @@ let DocumentIndex = class DocumentIndex extends Program {
             return done;
         };
         let joinListener;
+        let fetchedFirstForRemote = undefined;
         let updateDeferred;
         const signalUpdate = () => updateDeferred?.resolve();
-        const waitForUpdate = () => updateDeferred ? updateDeferred.promise : Promise.resolve();
-        if (typeof options?.remote === "object" && options?.remote.joining) {
+        const _waitForUpdate = () => updateDeferred ? updateDeferred.promise : Promise.resolve();
+        // ---------------- Live updates wiring (sorted-only with optional filter) ----------------
+        const normalizeUpdatesOption = (u) => {
+            if (u == null || u === false)
+                return undefined;
+            if (u === true)
+                return {
+                    merge: {
+                        filter: (evt) => evt,
+                    },
+                };
+            if (typeof u === "object") {
+                return {
+                    merge: u.merge
+                        ? {
+                            filter: typeof u.merge === "object" ? u.merge.filter : (evt) => evt,
+                        }
+                        : {},
+                };
+            }
+            return undefined;
+        };
+        const mergePolicy = normalizeUpdatesOption(options?.updates);
+        const hasLiveUpdates = mergePolicy !== undefined;
+        // sorted-only mode: no per-queue handling
+        // If live updates enabled, ensure deferred exists so awaiting paths can block until changes
+        if (hasLiveUpdates && !updateDeferred) {
+            updateDeferred = pDefer();
+        }
+        let updatesCleanup;
+        if (hasLiveUpdates) {
+            const localHash = this.node.identity.publicKey.hashcode();
+            if (mergePolicy?.merge) {
+                // Ensure local buffer exists for sorted merging
+                if (!peerBufferMap.has(localHash)) {
+                    peerBufferMap.set(localHash, { kept: 0, buffer: [] });
+                }
+            }
+            const onChange = async (evt) => {
+                // Optional filter to mutate/suppress change events
+                let filtered = evt.detail;
+                if (mergePolicy?.merge?.filter) {
+                    filtered = await mergePolicy.merge?.filter(evt.detail);
+                }
+                if (filtered) {
+                    // Remove entries that were deleted from all pending structures
+                    if (filtered.removed?.length) {
+                        // Remove from peer buffers
+                        for (const [_peer, entry] of peerBufferMap) {
+                            entry.buffer = entry.buffer.filter((x) => {
+                                const id = indexerTypes.toId(this.indexByResolver(x.indexed)).primitive;
+                                return !filtered.removed.some((r) => indexerTypes.toId(this.indexByResolver(r.__indexed))
+                                    .primitive === id);
+                            });
+                        }
+                        // no non-sorted queues in simplified mode
+                    }
+                    // Add new entries per strategy (sorted-only)
+                    if (filtered.added?.length) {
+                        const buf = peerBufferMap.get(localHash);
+                        for (const added of filtered.added) {
+                            const id = indexerTypes.toId(this.indexByResolver(added.__indexed)).primitive;
+                            if (visited.has(id))
+                                continue; // already presented
+                            visited.add(id);
+                            buf.buffer.push({
+                                value: (resolve ? added : added.__indexed),
+                                context: added.__context,
+                                from: this.node.identity.publicKey,
+                                indexed: coerceWithContext(added.__indexed, added.__context),
+                            });
+                        }
+                        buf.kept = buf.buffer.length;
+                    }
+                }
+                typeof options?.updates === "object" &&
+                    options?.updates?.onChange?.(evt.detail);
+                signalUpdate();
+            };
+            this.documentEvents.addEventListener("change", onChange);
+            updatesCleanup = () => {
+                this.documentEvents.removeEventListener("change", onChange);
+            };
+            const cleanupDefaultUpdates = cleanup;
+            cleanup = () => {
+                updatesCleanup?.();
+                return cleanupDefaultUpdates();
+            };
+        }
+        if (typeof options?.remote === "object" && options?.remote.wait) {
             // was used to account for missed results when a peer joins; omitted in this minimal handler
             updateDeferred = pDefer();
             // derive optional onMissedResults callback if provided
-            let onMissedResults = typeof options?.remote?.joining === "object" &&
-                typeof options?.remote.joining.onMissedResults === "function"
-                ? options.remote.joining.onMissedResults
+            let onMissedResults = typeof options?.remote?.wait === "object" &&
+                typeof options?.remote.onLateResults === "function"
+                ? options.remote.onLateResults
                 : undefined;
-            const waitForTime = typeof options.remote.joining === "object" &&
-                options.remote.joining.waitFor;
+            const waitForTime = typeof options.remote.wait === "object" && options.remote.wait.timeout;
             const prevMaybeSetDone = maybeSetDone;
             maybeSetDone = () => {
                 prevMaybeSetDone();
@@ -1495,16 +1713,25 @@ let DocumentIndex = class DocumentIndex extends Program {
                     signalUpdate();
                 }, waitForTime);
             controller.signal.addEventListener("abort", () => signalUpdate());
+            fetchedFirstForRemote = new Set();
             joinListener = this.attachJoinListener({
                 signal: controller.signal,
+                eager: options.remote.reach?.eager,
                 onPeer: async (pk) => {
                     if (done)
                         return;
                     const hash = pk.hashcode();
+                    await fetchPromise; // ensure fetches in flight are done
                     if (peerBufferMap.has(hash))
                         return;
+                    if (fetchedFirstForRemote.has(hash))
+                        return;
                     if (totalFetchedCounter > 0) {
-                        await fetchFirst(totalFetchedCounter, { from: [hash] });
+                        fetchPromise = fetchFirst(totalFetchedCounter, {
+                            from: [hash],
+                            fetchedFirstForRemote,
+                        });
+                        await fetchPromise;
                         if (onMissedResults) {
                             const pending = peerBufferMap.get(hash)?.buffer;
                             if (pending && pending.length > 0) {
@@ -1534,6 +1761,25 @@ let DocumentIndex = class DocumentIndex extends Program {
                 return cleanupDefault();
             };
         }
+        if (options?.closePolicy === "manual") {
+            let prevMaybeSetDone = maybeSetDone;
+            maybeSetDone = () => {
+                if (drain) {
+                    prevMaybeSetDone();
+                }
+            };
+        }
+        const remoteWaitActive = typeof options?.remote === "object" && !!options.remote.wait;
+        const waitForUpdateAndResetDeferred = async () => {
+            if (remoteWaitActive) {
+                // wait until: join fetch adds results, cleanup runs, or the join-wait times out
+                await _waitForUpdate();
+                // re-arm the deferred for the next cycle (only if joining is enabled and we're not done)
+                if (updateDeferred && !doneFn()) {
+                    updateDeferred = pDefer();
+                }
+            }
+        };
         return {
             close,
             next,
@@ -1546,24 +1792,22 @@ let DocumentIndex = class DocumentIndex extends Program {
                 return kept; // TODO this should be more accurate
             },
             all: async () => {
+                drain = true;
                 let result = [];
                 let c = 0;
                 while (doneFn() !== true) {
-                    c++;
                     let batch = await next(100);
-                    if (c > 100) {
-                        break;
+                    c += batch.length;
+                    if (c > WARNING_WHEN_ITERATING_FOR_MORE_THAN) {
+                        logger.warn("Iterating for more than " +
+                            WARNING_WHEN_ITERATING_FOR_MORE_THAN +
+                            " results");
                     }
                     if (batch.length > 0) {
                         result.push(...batch);
                         continue;
                     }
-                    // wait until: join fetch adds results, cleanup runs, or the join-wait times out
-                    await waitForUpdate();
-                    // re-arm the deferred for the next cycle (only if joining is enabled and we’re not done)
-                    if (updateDeferred && !doneFn()) {
-                        updateDeferred = pDefer();
-                    }
+                    await waitForUpdateAndResetDeferred();
                 }
                 cleanupAndDone();
                 return result;
@@ -1576,6 +1820,24 @@ let DocumentIndex = class DocumentIndex extends Program {
                 cleanupAndDone();
                 return batch[0];
             },
+            [Symbol.asyncIterator]: async function* () {
+                drain = true;
+                let c = 0;
+                while (doneFn() !== true) {
+                    const batch = await next(100);
+                    c += batch.length;
+                    if (c > WARNING_WHEN_ITERATING_FOR_MORE_THAN) {
+                        logger.warn("Iterating for more than " +
+                            WARNING_WHEN_ITERATING_FOR_MORE_THAN +
+                            " results");
+                    }
+                    for (const entry of batch) {
+                        yield entry;
+                    }
+                    await waitForUpdateAndResetDeferred();
+                }
+                cleanupAndDone();
+            },
         };
     }
     async updateResults(into, change, query, resolve) {
@@ -1659,17 +1921,12 @@ let DocumentIndex = class DocumentIndex extends Program {
         return all.map((x) => x.value);
     }
     async waitFor(other, options) {
-        await super.waitFor(other, options);
-        const ids = Array.isArray(other) ? other : [other];
-        const expectedHashes = new Set(ids.map((x) => typeof x === "string"
-            ? x
-            : x instanceof PublicSignKey
-                ? x.hashcode()
-                : getPublicKeyFromPeerId(x).hashcode()));
-        for (const key of expectedHashes) {
+        const hashes = await super.waitFor(other, options);
+        for (const key of hashes) {
             await waitFor(async () => (await this._log.replicationIndex.count({ query: { hash: key } })) >
                 0, options);
         }
+        return hashes;
     }
 };
 __decorate([