@peerbit/document 6.0.7 → 7.0.0-3a75d6e

package/dist/src/search.js

@@ -0,0 +1,845 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { field, serialize, variant } from "@dao-xyz/borsh";
+import { Cache } from "@peerbit/cache";
+import { PublicSignKey, sha256Base64Sync, } from "@peerbit/crypto";
+import * as types from "@peerbit/document-interface";
+import * as indexerTypes from "@peerbit/indexer-interface";
+import { HashmapIndex } from "@peerbit/indexer-simple";
+import { BORSH_ENCODING, Entry } from "@peerbit/log";
+import { logger as loggerFn } from "@peerbit/logger";
+import { Program } from "@peerbit/program";
+import { MissingResponsesError, RPC, queryAll, } from "@peerbit/rpc";
+import { SharedLog } from "@peerbit/shared-log";
+import { SilentDelivery } from "@peerbit/stream-interface";
+import { AbortError } from "@peerbit/time";
+import { concat, fromString } from "uint8arrays";
+import { copySerialization } from "./borsh.js";
+import { MAX_BATCH_SIZE } from "./constants.js";
+const logger = loggerFn({ module: "document-index" });
+let Operation = class Operation /* <T> */ {
+};
+Operation = __decorate([
+    variant(0)
+], Operation);
+export { Operation };
+export const BORSH_ENCODING_OPERATION = BORSH_ENCODING(Operation);
+/**
+ * Put a complete document at a key
+ */
+let PutOperation = class PutOperation extends Operation /* <T> */ {
+    data;
+    /* _value?: T; */
+    constructor(props) {
+        super();
+        this.data = props.data;
+    }
+};
+__decorate([
+    field({ type: Uint8Array }),
+    __metadata("design:type", Uint8Array)
+], PutOperation.prototype, "data", void 0);
+PutOperation = __decorate([
+    variant(0),
+    __metadata("design:paramtypes", [Object])
+], PutOperation);
+export { PutOperation };
+/* @variant(1)
+export class PutAllOperation<T> extends Operation<T> {
+    @field({ type: vec(PutOperation) })
+    docs: PutOperation<T>[];
+
+    constructor(props?: { docs: PutOperation<T>[] }) {
+        super();
+        if (props) {
+            this.docs = props.docs;
+        }
+    }
+}
+ */
+/**
+ * Delete a document at a key
+ */
+let DeleteOperation = class DeleteOperation extends Operation {
+    key;
+    constructor(props) {
+        super();
+        this.key = props.key;
+    }
+};
+__decorate([
+    field({ type: indexerTypes.IdKey }),
+    __metadata("design:type", indexerTypes.IdKey)
+], DeleteOperation.prototype, "key", void 0);
+DeleteOperation = __decorate([
+    variant(2),
+    __metadata("design:paramtypes", [Object])
+], DeleteOperation);
+export { DeleteOperation };
+const introduceEntries = async (responses, type, sync, options) => {
+    const results = [];
+    for (const response of responses) {
+        if (!response.from) {
+            logger.error("Missing from for response");
+        }
+        if (response.response instanceof types.Results) {
+            response.response.results.forEach((r) => r.init(type));
+            if (typeof options?.remote !== "boolean" && options?.remote?.sync) {
+                await sync(response.response);
+            }
+            options?.onResponse &&
+                (await options.onResponse(response.response, response.from)); // TODO fix types
+            results.push(response);
+        }
+        else if (response.response instanceof types.NoAccess) {
+            logger.error("Search resulted in access error");
+        }
+        else {
+            throw new Error("Unsupported");
+        }
+    }
+    return results;
+};
+const dedup = (allResult, dedupBy) => {
+    const unique = new Set();
+    const dedup = [];
+    for (const result of allResult) {
+        const key = indexerTypes.toId(dedupBy(result));
+        const primitive = key.primitive;
+        if (unique.has(primitive)) {
+            continue;
+        }
+        unique.add(primitive);
+        dedup.push(result);
+    }
+    return dedup;
+};
+const DEFAULT_INDEX_BY = "id";
+const isTransformerWithFunction = (options) => {
+    return options.transform != null;
+};
+let DocumentIndex = class DocumentIndex extends Program {
+    _query;
+    // Original document representation
+    documentType;
+    // transform options
+    transformer;
+    // The indexed document wrapped in a context
+    wrappedIndexedType;
+    // The database type, for recursive indexing
+    dbType;
+    indexedTypeIsDocumentType;
+    // Index key
+    indexBy;
+    indexByResolver;
+    index;
+    // Transformation, indexer
+    /* fields: IndexableFields<T, I>; */
+    _valueEncoding;
+    _sync;
+    _log;
+    _resolverProgramCache;
+    _resolverCache;
+    _isProgramValues;
+    _resultQueue;
+    constructor(properties) {
+        super();
+        this._query = properties?.query || new RPC();
+    }
+    get valueEncoding() {
+        return this._valueEncoding;
+    }
+    async open(properties) {
+        this._log = properties.log;
+        this.documentType = properties.documentType;
+        this.indexedTypeIsDocumentType =
+            !properties.transform?.type ||
+                properties.transform?.type === properties.documentType;
+        class IndexedClassWithContex {
+            __context;
+            constructor(value, context) {
+                Object.assign(this, value);
+                this.__context = context;
+            }
+        }
+        __decorate([
+            field({ type: types.Context }),
+            __metadata("design:type", types.Context)
+        ], IndexedClassWithContex.prototype, "__context", void 0);
+        // copy all prototype values from indexedType to IndexedClassWithContex
+        copySerialization((properties.transform?.type || properties.documentType), IndexedClassWithContex);
+        this.wrappedIndexedType = IndexedClassWithContex;
+        // if this.type is a class that extends Program we want to do special functionality
+        this._isProgramValues = this.documentType instanceof Program;
+        this.dbType = properties.dbType;
+        this._resultQueue = new Map();
+        this._sync = properties.sync;
+        const transformOptions = properties.transform;
+        this.transformer = transformOptions
+            ? isTransformerWithFunction(transformOptions)
+                ? (obj, context) => transformOptions.transform(obj, context)
+                : transformOptions.type
+                    ? (obj, context) => new transformOptions.type(obj, context)
+                    : (obj) => obj
+            : (obj) => obj; // TODO types
+        const maybeArr = properties.indexBy || DEFAULT_INDEX_BY;
+        this.indexBy = Array.isArray(maybeArr) ? maybeArr : [maybeArr];
+        this.indexByResolver = (obj) => indexerTypes.extractFieldValue(obj, this.indexBy);
+        this._valueEncoding = BORSH_ENCODING(this.documentType);
+        if (this._isProgramValues) {
+            this._resolverProgramCache = new Map();
+        }
+        this._resolverCache = new Cache({ max: 10 }); // TODO choose limit better (adaptive)
+        this.index =
+            (await (await this.node.indexer.scope(sha256Base64Sync(concat([this._log.log.id, fromString("/document-index")])))).init({
+                indexBy: this.indexBy,
+                schema: this.wrappedIndexedType,
+                nested: {
+                    match: (obj) => obj instanceof this.dbType,
+                    query: async (obj, query) => obj.index.search(query),
+                },
+                /* maxBatchSize: MAX_BATCH_SIZE */
+            })) || new HashmapIndex();
+        await this._query.open({
+            topic: sha256Base64Sync(concat([this._log.log.id, fromString("/document")])),
+            responseHandler: async (query, ctx) => {
+                if (!ctx.from) {
+                    logger.info("Receieved query without from");
+                    return;
+                }
+                if (properties.canSearch &&
+                    (query instanceof indexerTypes.SearchRequest ||
+                        query instanceof indexerTypes.CollectNextRequest) &&
+                    !(await properties.canSearch(query, ctx.from))) {
+                    return new types.NoAccess();
+                }
+                if (query instanceof indexerTypes.CloseIteratorRequest) {
+                    this.processCloseIteratorRequest(query, ctx.from);
+                }
+                else {
+                    const results = await this.processQuery(query, ctx.from, false, {
+                        canRead: properties.canRead,
+                    });
+                    return new types.Results({
+                        // Even if results might have length 0, respond, because then we now at least there are no matching results
+                        results: results.results,
+                        kept: results.kept,
+                    });
+                }
+            },
+            responseType: types.AbstractSearchResult,
+            queryType: indexerTypes.AbstractSearchRequest,
+        });
+    }
+    getPending(cursorId) {
+        const queue = this._resultQueue.get(cursorId);
+        if (queue) {
+            return queue.queue.length + queue.keptInIndex;
+        }
+        return this.index.getPending(cursorId);
+    }
+    async close(from) {
+        const closed = await super.close(from);
+        if (closed) {
+            await this.index.stop?.();
+        }
+        return closed;
+    }
+    async drop(from) {
+        const dropped = await super.drop(from);
+        if (dropped) {
+            await this.index.drop?.();
+            await this.index.stop?.();
+        }
+        return dropped;
+    }
+    async get(key, options) {
+        return (await this.getDetailed(key instanceof indexerTypes.IdKey ? key : indexerTypes.toId(key), options))?.[0]?.results[0]?.value;
+    }
+    async put(value, entry, id) {
+        const idString = id.primitive;
+        if (this._isProgramValues) {
+            this._resolverProgramCache.set(idString, value);
+        }
+        else {
+            this._resolverCache.add(idString, value);
+        }
+        const existing = await this.index.get(id);
+        const context = new types.Context({
+            created: existing?.value.__context.created ||
+                entry.meta.clock.timestamp.wallTime,
+            modified: entry.meta.clock.timestamp.wallTime,
+            head: entry.hash,
+            gid: entry.gid,
+            size: entry.payloadByteLength,
+        });
+        const valueToIndex = await this.transformer(value, context);
+        const wrappedValueToIndex = new this.wrappedIndexedType(valueToIndex, context);
+        await this.index.put(wrappedValueToIndex);
+    }
+    del(key) {
+        if (this._isProgramValues) {
+            this._resolverProgramCache.delete(key.primitive);
+        }
+        else {
+            this._resolverCache.del(key.primitive);
+        }
+        return this.index.del(new indexerTypes.DeleteRequest({
+            query: [indexerTypes.getMatcher(this.indexBy, key.key)],
+        }));
+    }
+    async getDetailed(key, options) {
+        let results;
+        if (key instanceof Uint8Array) {
+            results = await this.queryDetailed(new indexerTypes.SearchRequest({
+                query: [
+                    new indexerTypes.ByteMatchQuery({ key: this.indexBy, value: key }),
+                ],
+            }), options);
+        }
+        else {
+            const indexableKey = indexerTypes.toIdeable(key);
+            if (typeof indexableKey === "number" ||
+                typeof indexableKey === "bigint") {
+                results = await this.queryDetailed(new indexerTypes.SearchRequest({
+                    query: [
+                        new indexerTypes.IntegerCompare({
+                            key: this.indexBy,
+                            compare: indexerTypes.Compare.Equal,
+                            value: indexableKey,
+                        }),
+                    ],
+                }), options);
+            }
+            else if (typeof indexableKey === "string") {
+                results = await this.queryDetailed(new indexerTypes.SearchRequest({
+                    query: [
+                        new indexerTypes.StringMatch({
+                            key: this.indexBy,
+                            value: indexableKey,
+                        }),
+                    ],
+                }), options);
+            }
+            else if (indexableKey instanceof Uint8Array) {
+                results = await this.queryDetailed(new indexerTypes.SearchRequest({
+                    query: [
+                        new indexerTypes.ByteMatchQuery({
+                            key: this.indexBy,
+                            value: indexableKey,
+                        }),
+                    ],
+                }), options);
+            }
+        }
+        return results;
+    }
+    getSize() {
+        return this.index.getSize();
+    }
+    async resolveDocument(value) {
+        const cached = this._resolverCache.get(value.id.primitive) ||
+            this._resolverProgramCache?.get(value.id.primitive);
+        if (cached != null) {
+            return { value: cached };
+        }
+        if (this.indexedTypeIsDocumentType) {
+            // cast value to T, i.e. convert the class but keep all properties except the __context
+            const obj = Object.assign(Object.create(this.documentType.prototype), value.value);
+            delete obj.__context;
+            return { value: obj };
+        }
+        const head = await this._log.log.get(value.value.__context.head);
+        if (!head) {
+            return undefined; // we could end up here if we recently pruned the document and other peers never persisted the entry
+            // TODO update changes in index before removing entries from log entry storage
+        }
+        const payloadValue = await head.getPayloadValue();
+        if (payloadValue instanceof PutOperation) {
+            return {
+                value: this.valueEncoding.decoder(payloadValue.data),
+                /* size: payloadValue.data.byteLength */
+            };
+        }
+        throw new Error("Unexpected value type when getting document: " +
+            payloadValue?.constructor?.name || typeof payloadValue);
+    }
+    async processQuery(query, from, isLocal, options) {
+        // We do special case for querying the id as we can do it faster than iterating
+        let prevQueued = isLocal
+            ? undefined
+            : this._resultQueue.get(query.idString);
+        if (prevQueued && !from.equals(prevQueued.from)) {
+            throw new Error("Different from in queued results");
+        }
+        let indexedResult = undefined;
+        if (query instanceof indexerTypes.SearchRequest) {
+            indexedResult = await this.index.query(query);
+        }
+        else if (query instanceof indexerTypes.CollectNextRequest) {
+            indexedResult =
+                prevQueued?.keptInIndex === 0
+                    ? { kept: 0, results: [] }
+                    : await this.index.next(query);
+        }
+        else {
+            throw new Error("Unsupported");
+        }
+        const filteredResults = [];
+        let resultSize = 0;
+        let toIterate = prevQueued
+            ? [...prevQueued.queue, ...indexedResult.results]
+            : indexedResult.results;
+        if (prevQueued) {
+            this._resultQueue.delete(query.idString);
+            prevQueued = undefined;
+        }
+        if (!isLocal) {
+            prevQueued = {
+                from,
+                queue: [],
+                timeout: setTimeout(() => {
+                    this._resultQueue.delete(query.idString);
+                }, 6e4),
+                keptInIndex: indexedResult.kept,
+            };
+            this._resultQueue.set(query.idString, prevQueued);
+        }
+        for (const result of toIterate) {
+            if (!isLocal) {
+                resultSize += result.value.__context.size;
+                if (resultSize > MAX_BATCH_SIZE) {
+                    prevQueued.queue.push(result);
+                    continue;
+                }
+            }
+            const value = await this.resolveDocument(result);
+            if (!value ||
+                (options?.canRead && !(await options.canRead(value.value, from)))) {
+                continue;
+            }
+            filteredResults.push(new types.ResultWithSource({
+                context: result.value.__context,
+                value: value.value,
+                source: serialize(value.value),
+                indexed: result.value,
+            }));
+        }
+        const results = new types.Results({
+            results: filteredResults,
+            kept: BigInt(indexedResult.kept + (prevQueued?.queue.length || 0)),
+        });
+        if (!isLocal && results.kept === 0n) {
+            this.clearResultsQueue(query);
+        }
+        return results;
+    }
+    clearResultsQueue(query) {
+        const queue = this._resultQueue.get(query.idString);
+        if (queue) {
+            clearTimeout(queue.timeout);
+            this._resultQueue.delete(query.idString);
+        }
+    }
+    async processCloseIteratorRequest(query, publicKey) {
+        const queueData = this._resultQueue.get(query.idString);
+        if (queueData && !queueData.from.equals(publicKey)) {
+            logger.info("Ignoring close iterator request from different peer");
+            return;
+        }
+        this.clearResultsQueue(query);
+        return this.index.close(query);
+    }
+    /**
+     * Query and retrieve results with most details
+     * @param queryRequest
+     * @param options
+     * @returns
+     */
+    async queryDetailed(queryRequest, options) {
+        const local = typeof options?.local === "boolean" ? options?.local : true;
+        let remote = undefined;
+        if (typeof options?.remote === "boolean") {
+            if (options?.remote) {
+                remote = {};
+            }
+            else {
+                remote = undefined;
+            }
+        }
+        else {
+            remote = options?.remote || {};
+        }
+        if (remote && remote.priority == null) {
+            // give queries higher priority than other "normal" data activities
+            // without this, we might have a scenario that a peer joina  network with large amount of data to be synced, but can not query anything before that is done
+            // this will lead to bad UX as you usually want to list/expore whats going on before doing any replication work
+            remote.priority = 1;
+        }
+        const promises = [];
+        if (!local && !remote) {
+            throw new Error("Expecting either 'options.remote' or 'options.local' to be true");
+        }
+        const allResults = [];
+        if (local) {
+            const results = await this.processQuery(queryRequest, this.node.identity.publicKey, true);
+            if (results.results.length > 0) {
+                options?.onResponse &&
+                    (await options.onResponse(results, this.node.identity.publicKey));
+                allResults.push(results);
+            }
+        }
+        if (remote) {
+            const replicatorGroups = await this._log.getReplicatorUnion(remote.minAge);
+            if (replicatorGroups) {
+                const groupHashes = replicatorGroups.map((x) => [x]);
+                const fn = async () => {
+                    const rs = [];
+                    const responseHandler = async (results) => {
+                        for (const r of await introduceEntries(results, this.documentType, this._sync, options)) {
+                            rs.push(r.response);
+                        }
+                    };
+                    try {
+                        if (queryRequest instanceof indexerTypes.CloseIteratorRequest) {
+                            // don't wait for responses
+                            await this._query.request(queryRequest, { mode: remote.mode });
+                        }
+                        else {
+                            await queryAll(this._query, groupHashes, queryRequest, responseHandler, remote);
+                        }
+                    }
+                    catch (error) {
+                        if (error instanceof MissingResponsesError) {
+                            logger.warn("Did not reciveve responses from all shard");
+                            if (remote?.throwOnMissing) {
+                                throw error;
+                            }
+                        }
+                        else {
+                            throw error;
+                        }
+                    }
+                    return rs;
+                };
+                promises.push(fn());
+            }
+            else {
+                // TODO send without direction out to the world? or just assume we can insert?
+                /* 	promises.push(
+                        this._query
+                            .request(queryRequest, remote)
+                            .then((results) => introduceEntries(results, this.type, this._sync, options).then(x => x.map(y => y.response)))
+                    ); */
+                /* throw new Error(
+                    "Missing remote replicator info for performing distributed document query"
+                ); */
+            }
+        }
+        const resolved = await Promise.all(promises);
+        for (const r of resolved) {
+            if (r) {
+                if (r instanceof Array) {
+                    allResults.push(...r);
+                }
+                else {
+                    allResults.push(r);
+                }
+            }
+        }
+        return allResults;
+    }
+    /**
+     * Query and retrieve results
+     * @param queryRequest
+     * @param options
+     * @returns
+     */
+    async search(queryRequest, options) {
+        // Set fetch to search size, or max value (default to max u32 (4294967295))
+        queryRequest.fetch = queryRequest.fetch ?? 0xffffffff;
+        // So that the iterator is pre-fetching the right amount of entries
+        const iterator = this.iterate(queryRequest, options);
+        // So that this call will not do any remote requests
+        const allResults = [];
+        while (iterator.done() === false &&
+            queryRequest.fetch > allResults.length) {
+            // We might need to pull .next multiple time due to data message size limitations
+            for (const result of await iterator.next(queryRequest.fetch - allResults.length)) {
+                allResults.push(result);
+            }
+        }
+        await iterator.close();
+        //s Deduplicate and return values directly
+        return dedup(allResults, this.indexByResolver);
+    }
+    /**
+     * Query and retrieve documents in a iterator
+     * @param queryRequest
+     * @param options
+     * @returns
+     */
+    iterate(queryRequest, options) {
+        let fetchPromise = undefined;
+        const peerBufferMap = new Map();
+        const visited = new Set();
+        let done = false;
+        let first = false;
+        // TODO handle join/leave while iterating
+        const controller = new AbortController();
+        const peerBuffers = () => {
+            return [...peerBufferMap.values()].map((x) => x.buffer).flat();
+        };
+        const fetchFirst = async (n) => {
+            done = true; // Assume we are donne
+            queryRequest.fetch = n;
+            await this.queryDetailed(queryRequest, {
+                ...options,
+                onResponse: async (response, from) => {
+                    if (!from) {
+                        logger.error("Missing response from");
+                        return;
+                    }
+                    if (response instanceof types.NoAccess) {
+                        logger.error("Dont have access");
+                        return;
+                    }
+                    else if (response instanceof types.Results) {
+                        const results = response;
+                        if (results.kept === 0n && results.results.length === 0) {
+                            return;
+                        }
+                        if (results.kept > 0n) {
+                            done = false; // we have more to do later!
+                        }
+                        const buffer = [];
+                        for (const result of results.results) {
+                            const indexKey = indexerTypes.toId(this.indexByResolver(result.value)).primitive;
+                            if (visited.has(indexKey)) {
+                                continue;
+                            }
+                            visited.add(indexKey);
+                            buffer.push({
+                                value: result.value,
+                                context: result.context,
+                                from,
+                                indexed: result.indexed ||
+                                    (await this.transformer(result.value, result.context)),
+                            });
+                        }
+                        peerBufferMap.set(from.hashcode(), {
+                            buffer,
+                            kept: Number(response.kept),
+                        });
+                    }
+                    else {
+                        throw new Error("Unsupported result type: " + response?.constructor?.name);
+                    }
+                },
+            });
+            if (done) {
+                this.clearResultsQueue(queryRequest);
+            }
+            return done;
+        };
+        const fetchAtLeast = async (n) => {
+            if (done && first) {
+                return;
+            }
+            await fetchPromise;
+            if (!first) {
+                first = true;
+                fetchPromise = fetchFirst(n);
+                return fetchPromise;
+            }
+            const promises = [];
+            let resultsLeft = 0;
+            for (const [peer, buffer] of peerBufferMap) {
+                if (buffer.buffer.length < n) {
+                    if (buffer.kept === 0) {
+                        if (peerBufferMap.get(peer)?.buffer.length === 0) {
+                            peerBufferMap.delete(peer); // No more results
+                        }
+                        continue;
+                    }
+                    // TODO buffer more than deleted?
+                    // TODO batch to multiple 'to's
+                    const collectRequest = new indexerTypes.CollectNextRequest({
+                        id: queryRequest.id,
+                        amount: n - buffer.buffer.length,
+                    });
+                    // Fetch locally?
+                    if (peer === this.node.identity.publicKey.hashcode()) {
+                        promises.push(this.processQuery(collectRequest, this.node.identity.publicKey, true)
+                            .then(async (results) => {
+                            resultsLeft += Number(results.kept);
+                            if (results.results.length === 0) {
+                                if (peerBufferMap.get(peer)?.buffer.length === 0) {
+                                    peerBufferMap.delete(peer); // No more results
+                                }
+                            }
+                            else {
+                                const peerBuffer = peerBufferMap.get(peer);
+                                if (!peerBuffer) {
+                                    return;
+                                }
+                                peerBuffer.kept = Number(results.kept);
+                                for (const result of results.results) {
+                                    if (visited.has(indexerTypes.toId(this.indexByResolver(result.value))
+                                        .primitive)) {
+                                        continue;
+                                    }
+                                    visited.add(indexerTypes.toId(this.indexByResolver(result.value))
+                                        .primitive);
+                                    peerBuffer.buffer.push({
+                                        value: result.value,
+                                        context: result.context,
+                                        from: this.node.identity.publicKey,
+                                        indexed: result.indexed ||
+                                            (await this.transformer(result.value, result.context)),
+                                    });
+                                }
+                            }
+                        })
+                            .catch((e) => {
+                            logger.error("Failed to collect sorted results from self. " + e?.message);
+                            peerBufferMap.delete(peer);
+                        }));
+                    }
+                    else {
+                        // Fetch remotely
+                        promises.push(this._query
+                            .request(collectRequest, {
+                            ...options,
+                            signal: controller.signal,
+                            priority: 1,
+                            mode: new SilentDelivery({ to: [peer], redundancy: 1 }),
+                        })
+                            .then((response) => introduceEntries(response, this.documentType, this._sync, options)
+                            .then((responses) => {
+                            responses.map((response) => {
+                                resultsLeft += Number(response.response.kept);
+                                if (!response.from) {
+                                    logger.error("Missing from for sorted query");
+                                    return;
+                                }
+                                if (response.response.results.length === 0) {
+                                    if (peerBufferMap.get(peer)?.buffer.length === 0) {
+                                        peerBufferMap.delete(peer); // No more results
+                                    }
+                                }
+                                else {
+                                    const peerBuffer = peerBufferMap.get(peer);
+                                    if (!peerBuffer) {
+                                        return;
+                                    }
+                                    peerBuffer.kept = Number(response.response.kept);
+                                    for (const result of response.response.results) {
+                                        if (visited.has(indexerTypes.toId(this.indexByResolver(result.value)).primitive)) {
+                                            continue;
+                                        }
+                                        visited.add(indexerTypes.toId(this.indexByResolver(result.value)).primitive);
+                                        peerBuffer.buffer.push({
+                                            value: result.value,
+                                            context: result.context,
+                                            from: response.from,
+                                            indexed: this.transformer(result.value, result.context),
+                                        });
+                                    }
+                                }
+                            });
+                        })
+                            .catch((e) => {
+                            logger.error("Failed to collect sorted results from: " +
+                                peer +
+                                ". " +
+                                e?.message);
+                            peerBufferMap.delete(peer);
+                        })));
+                    }
+                }
+                else {
+                    resultsLeft += peerBufferMap.get(peer)?.kept || 0;
+                }
+            }
+            return (fetchPromise = Promise.all(promises).then(() => {
+                return resultsLeft === 0; // 0 results left to fetch and 0 pending results
+            }));
+        };
+        const next = async (n) => {
+            if (n < 0) {
+                throw new Error("Expecting to fetch a positive amount of element");
+            }
+            if (n === 0) {
+                return [];
+            }
+            // TODO everything below is not very optimized
+            const fetchedAll = await fetchAtLeast(n);
+            // get n next top entries, shift and pull more results
+            const peerBuffersArr = peerBuffers();
+            const results = peerBuffersArr.sort((a, b) => indexerTypes.extractSortCompare(a.indexed, b.indexed, queryRequest.sort));
+            const pendingMoreResults = n < results.length;
+            const batch = results.splice(0, n);
+            for (const result of batch) {
+                const arr = peerBufferMap.get(result.from.hashcode());
+                if (!arr) {
+                    logger.error("Unexpected empty result buffer");
+                    continue;
+                }
+                const idx = arr.buffer.findIndex((x) => x.value === result.value);
+                if (idx >= 0) {
+                    arr.buffer.splice(idx, 1);
+                }
+            }
+            done = fetchedAll && !pendingMoreResults;
+            return dedup(batch.map((x) => x.value), this.indexByResolver);
+        };
+        const close = async () => {
+            controller.abort(new AbortError("Iterator closed"));
+            const closeRequest = new indexerTypes.CloseIteratorRequest({
+                id: queryRequest.id,
+            });
+            const promises = [];
+            for (const [peer, buffer] of peerBufferMap) {
+                if (buffer.kept === 0) {
+                    peerBufferMap.delete(peer);
+                    continue;
+                }
+                // Fetch locally?
+                if (peer === this.node.identity.publicKey.hashcode()) {
+                    promises.push(this.processCloseIteratorRequest(closeRequest, this.node.identity.publicKey));
+                }
+                else {
+                    // Close remote
+                    promises.push(this._query.send(closeRequest, {
+                        ...options,
+                        mode: new SilentDelivery({ to: [peer], redundancy: 1 }),
+                    }));
+                }
+            }
+            await Promise.all(promises);
+        };
+        return {
+            close,
+            next,
+            done: () => done,
+        };
+    }
+};
+__decorate([
+    field({ type: RPC }),
+    __metadata("design:type", RPC)
+], DocumentIndex.prototype, "_query", void 0);
+DocumentIndex = __decorate([
+    variant("documents_index"),
+    __metadata("design:paramtypes", [Object])
+], DocumentIndex);
+export { DocumentIndex };
+//# sourceMappingURL=search.js.map