@peerbit/document 6.0.7 → 7.0.0-3a75d6e

package/lib/esm/document-index.js

@@ -1,942 +0,0 @@
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-var __metadata = (this && this.__metadata) || function (k, v) {
-    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
-};
-import { field, variant } from "@dao-xyz/borsh";
-import { asString } from "./utils.js";
-import { BORSH_ENCODING } from "@peerbit/log";
-import { equals } from "@peerbit/uint8arrays";
-import { Program } from "@peerbit/program";
-import { IntegerCompare, ByteMatchQuery, StringMatch, ResultWithSource, StateFieldQuery, compare, MissingField, StringMatchMethod, LogicalQuery, And, Or, BoolQuery, CollectNextRequest, AbstractSearchRequest, SearchRequest, SortDirection, CloseIteratorRequest, NoAccess, AbstractSearchResult } from "./query.js";
-import { RPC, queryAll, MissingResponsesError } from "@peerbit/rpc";
-import { Results } from "./query.js";
-import { logger as loggerFn } from "@peerbit/logger";
-import { Cache } from "@peerbit/cache";
-import { sha256Base64Sync } from "@peerbit/crypto";
-import { concat, fromString } from "uint8arrays";
-import { SilentDelivery } from "@peerbit/stream-interface";
-import { AbortError } from "@peerbit/time";
-const logger = loggerFn({ module: "document-index" });
-const stringArraysEquals = (a, b) => {
-    if (a.length !== b.length) {
-        return false;
-    }
-    for (let i = 0; i < a.length; i++) {
-        if (a[i] !== b[i]) {
-            return false;
-        }
-    }
-    return true;
-};
-let Operation = class Operation {
-};
-Operation = __decorate([
-    variant(0)
-], Operation);
-export { Operation };
-export const BORSH_ENCODING_OPERATION = BORSH_ENCODING(Operation);
-let PutOperation = class PutOperation extends Operation {
-    key;
-    data;
-    _value;
-    constructor(props) {
-        super();
-        if (props) {
-            this.key = props.key;
-            this.data = props.data;
-            this._value = props.value;
-        }
-    }
-    get value() {
-        if (!this._value) {
-            throw new Error("Value not decoded, invoke getValue(...) once");
-        }
-        return this._value;
-    }
-    getValue(encoding) {
-        if (this._value) {
-            return this._value;
-        }
-        this._value = encoding.decoder(this.data);
-        return this._value;
-    }
-};
-__decorate([
-    field({ type: "string" }),
-    __metadata("design:type", String)
-], PutOperation.prototype, "key", void 0);
-__decorate([
-    field({ type: Uint8Array }),
-    __metadata("design:type", Uint8Array)
-], PutOperation.prototype, "data", void 0);
-PutOperation = __decorate([
-    variant(0),
-    __metadata("design:paramtypes", [Object])
-], PutOperation);
-export { PutOperation };
-/* @variant(1)
-export class PutAllOperation<T> extends Operation<T> {
-    @field({ type: vec(PutOperation) })
-    docs: PutOperation<T>[];
-
-    constructor(props?: { docs: PutOperation<T>[] }) {
-        super();
-        if (props) {
-            this.docs = props.docs;
-        }
-    }
-}
- */
-let DeleteOperation = class DeleteOperation extends Operation {
-    key;
-    constructor(props) {
-        super();
-        if (props) {
-            this.key = props.key;
-        }
-    }
-};
-__decorate([
-    field({ type: "string" }),
-    __metadata("design:type", String)
-], DeleteOperation.prototype, "key", void 0);
-DeleteOperation = __decorate([
-    variant(2),
-    __metadata("design:paramtypes", [Object])
-], DeleteOperation);
-export { DeleteOperation };
-const sortCompare = (av, bv) => {
-    if (typeof av === "string" && typeof bv === "string") {
-        return av.localeCompare(bv);
-    }
-    if (av < bv) {
-        return -1;
-    }
-    else if (av > bv) {
-        return 1;
-    }
-    return 0;
-};
-const extractFieldValue = (doc, path) => {
-    for (let i = 0; i < path.length; i++) {
-        doc = doc[path[i]];
-    }
-    return doc;
-};
-const extractSortCompare = (a, b, sorts) => {
-    for (const sort of sorts) {
-        const av = extractFieldValue(a, sort.key);
-        const bv = extractFieldValue(b, sort.key);
-        const cmp = sortCompare(av, bv);
-        if (cmp != 0) {
-            if (sort.direction === SortDirection.ASC) {
-                return cmp;
-            }
-            else {
-                return -cmp;
-            }
-        }
-    }
-    return 0;
-};
-const resolvedSort = async (arr, index, sorts) => {
-    await Promise.all(arr.map(async (result) => (result[SORT_TMP_KEY] = await index(result.value.value, result.context))));
-    arr.sort((a, b) => extractSortCompare(a[SORT_TMP_KEY], b[SORT_TMP_KEY], sorts));
-    return arr;
-};
-const SORT_TMP_KEY = "__sort_ref";
-const introduceEntries = async (responses, type, sync, options) => {
-    const results = [];
-    for (const response of responses) {
-        if (!response.from) {
-            logger.error("Missing from for response");
-        }
-        if (response.response instanceof Results) {
-            response.response.results.forEach((r) => r.init(type));
-            if (typeof options?.remote !== "boolean" && options?.remote?.sync) {
-                await sync(response.response);
-            }
-            options?.onResponse &&
-                options.onResponse(response.response, response.from); // TODO fix types
-            results.push(response);
-        }
-        else if (response.response instanceof NoAccess) {
-            logger.error("Search resulted in access error");
-        }
-        else {
-            throw new Error("Unsupported");
-        }
-    }
-    return results;
-};
-const dedup = (allResult, dedupBy) => {
-    const unique = new Set();
-    const dedup = [];
-    for (const result of allResult) {
-        const key = asString(dedupBy(result));
-        if (unique.has(key)) {
-            continue;
-        }
-        unique.add(key);
-        dedup.push(result);
-    }
-    return dedup;
-};
-const DEFAULT_INDEX_BY = "id";
-/*
-if (!(await this.canRead(message.sender))) {
-    throw new AccessError();
-} */
-export const MAX_DOCUMENT_SIZE = 5e6;
-const getBatchFromResults = (results, wantedSize, maxSize = MAX_DOCUMENT_SIZE) => {
-    const batch = [];
-    let size = 0;
-    for (const result of results) {
-        batch.push(result);
-        size += result.value.last.data.length;
-        if (size > maxSize) {
-            break;
-        }
-        if (wantedSize <= batch.length) {
-            break;
-        }
-    }
-    results.splice(0, batch.length);
-    return batch;
-};
-let DocumentIndex = class DocumentIndex extends Program {
-    _query;
-    type;
-    dbType;
-    // Index key
-    _indexBy;
-    _indexByArr;
-    // Resolve doc value by index key
-    indexByResolver;
-    // Indexed (transforms an docuemnt into an obj with fields that ought to be indexed)
-    _toIndex;
-    _valueEncoding;
-    _sync;
-    _index;
-    _resultsCollectQueue;
-    _log;
-    constructor(properties) {
-        super();
-        this._query = properties?.query || new RPC();
-    }
-    get index() {
-        return this._index;
-    }
-    get valueEncoding() {
-        return this._valueEncoding;
-    }
-    get toIndex() {
-        return this._toIndex;
-    }
-    async open(properties) {
-        this._index = new Map();
-        this._log = properties.log;
-        this.type = properties.type;
-        this.dbType = properties.dbType;
-        this._sync = properties.sync;
-        this._toIndex = properties.fields;
-        this._indexBy = properties.indexBy || DEFAULT_INDEX_BY;
-        this._indexByArr = Array.isArray(this._indexBy)
-            ? this._indexBy
-            : [this._indexBy];
-        this.indexByResolver =
-            typeof this._indexBy === "string"
-                ? (obj) => obj[this._indexBy]
-                : (obj) => extractFieldValue(obj, this._indexBy);
-        this._valueEncoding = BORSH_ENCODING(this.type);
-        this._resultsCollectQueue = new Cache({ max: 10000 }); // TODO choose limit better
-        await this._query.open({
-            topic: sha256Base64Sync(concat([this._log.log.id, fromString("/document")])),
-            responseHandler: async (query, ctx) => {
-                if (!ctx.from) {
-                    logger.info("Receieved query without from");
-                    return;
-                }
-                if (properties.canSearch &&
-                    (query instanceof SearchRequest ||
-                        query instanceof CollectNextRequest) &&
-                    !(await properties.canSearch(query, ctx.from))) {
-                    return new NoAccess();
-                }
-                if (query instanceof CloseIteratorRequest) {
-                    this.processCloseIteratorRequest(query, ctx.from);
-                }
-                else {
-                    const results = await this.processFetchRequest(query, ctx.from, {
-                        canRead: properties.canRead
-                    });
-                    return new Results({
-                        // Even if results might have length 0, respond, because then we now at least there are no matching results
-                        results: results.results.map((r) => {
-                            if (r.value.last instanceof PutOperation === false) {
-                                throw new Error("Unexpected value type on local results: " +
-                                    r.value.last?.constructor.name ||
-                                    typeof r.value.last);
-                            }
-                            return new ResultWithSource({
-                                source: r.value.last.data,
-                                context: r.context
-                            });
-                        }),
-                        kept: BigInt(results.kept)
-                    });
-                }
-            },
-            responseType: AbstractSearchResult,
-            queryType: AbstractSearchRequest
-        });
-    }
-    async get(key, options) {
-        return (await this.getDetailed(key, options))?.[0]?.results[0]?.value;
-    }
-    async getDetailed(key, options) {
-        let results;
-        if (key instanceof Uint8Array) {
-            results = await this.queryDetailed(new SearchRequest({
-                query: [new ByteMatchQuery({ key: this._indexByArr, value: key })]
-            }), options);
-        }
-        else {
-            const stringValue = asString(key);
-            results = await this.queryDetailed(new SearchRequest({
-                query: [
-                    new StringMatch({
-                        key: this._indexByArr,
-                        value: stringValue
-                    })
-                ]
-            }), options);
-        }
-        return results;
-    }
-    get size() {
-        return this._index.size;
-    }
-    async getDocumentWithLastOperation(value) {
-        if (value.reference) {
-            return value.reference;
-        }
-        const head = await await this._log.log.get(value.context.head);
-        if (!head) {
-            return undefined; // we could end up here if we recently pruned the document and other peers never persisted the entry
-            // TODO update changes in index before removing entries from log entry storage
-        }
-        const payloadValue = await head.getPayloadValue();
-        if (payloadValue instanceof PutOperation) {
-            return {
-                value: payloadValue.getValue(this.valueEncoding),
-                last: payloadValue
-            };
-        }
-        throw new Error("Unexpected value type when getting document: " +
-            payloadValue?.constructor?.name || typeof payloadValue);
-    }
-    getDocument(value) {
-        return this.getDocumentWithLastOperation(value).then((r) => r?.value);
-    }
-    async _queryDocuments(filter) {
-        // Whether we return the full operation data or just the db value
-        const results = [];
-        for (const value of this._index.values()) {
-            if (await filter(value)) {
-                const topDoc = await this.getDocumentWithLastOperation(value);
-                topDoc &&
-                    results.push({
-                        context: value.context,
-                        value: topDoc
-                    });
-            }
-        }
-        return results;
-    }
-    async processFetchRequest(query, from, options) {
-        // We do special case for querying the id as we can do it faster than iterating
-        if (query instanceof SearchRequest) {
-            // Special case querying ids
-            if (query.query.length === 1 &&
-                (query.query[0] instanceof ByteMatchQuery ||
-                    query.query[0] instanceof StringMatch) &&
-                stringArraysEquals(query.query[0].key, this._indexByArr)) {
-                const firstQuery = query.query[0];
-                if (firstQuery instanceof ByteMatchQuery) {
-                    const doc = this._index.get(asString(firstQuery.value));
-                    const topDoc = doc && (await this.getDocumentWithLastOperation(doc));
-                    return topDoc
-                        ? {
-                            results: [
-                                {
-                                    value: topDoc,
-                                    context: doc.context
-                                }
-                            ],
-                            kept: 0
-                        }
-                        : { results: [], kept: 0 };
-                }
-                else if (firstQuery instanceof StringMatch &&
-                    firstQuery.method === StringMatchMethod.exact &&
-                    firstQuery.caseInsensitive === false) {
-                    const doc = this._index.get(firstQuery.value);
-                    const topDoc = doc && (await this.getDocumentWithLastOperation(doc));
-                    return topDoc
-                        ? {
-                            results: [
-                                {
-                                    value: topDoc,
-                                    context: doc.context
-                                }
-                            ],
-                            kept: 0
-                        }
-                        : { results: [], kept: 0 };
-                }
-            }
-            // Handle query normally
-            let results = await this._queryDocuments(async (doc) => {
-                for (const f of query.query) {
-                    if (!(await this.handleQueryObject(f, doc))) {
-                        return false;
-                    }
-                }
-                return true;
-            });
-            if (options?.canRead) {
-                const keepFilter = await Promise.all(results.map((x) => options?.canRead(x.value.value, from)));
-                results = results.filter((x, i) => keepFilter[i]);
-            }
-            // Sort
-            await resolvedSort(results, this._toIndex, query.sort);
-            const batch = getBatchFromResults(results, query.fetch);
-            if (results.length > 0) {
-                this._resultsCollectQueue.add(query.idString, {
-                    arr: results,
-                    from
-                }); // cache resulst not returned
-            }
-            return { results: batch, kept: results.length }; // Only return 1 result since we are doing distributed sort, TODO buffer more initially
-        }
-        else if (query instanceof CollectNextRequest) {
-            const results = this._resultsCollectQueue.get(query.idString);
-            if (!results) {
-                return {
-                    results: [],
-                    kept: 0
-                };
-            }
-            const batch = getBatchFromResults(results.arr, query.amount);
-            if (results.arr.length === 0) {
-                this._resultsCollectQueue.del(query.idString); // TODO add tests for proper cleanup/timeouts
-            }
-            return { results: batch, kept: results.arr.length };
-        }
-        throw new Error("Unsupported");
-    }
-    async processCloseIteratorRequest(query, publicKey) {
-        const entry = this._resultsCollectQueue.get(query.idString);
-        if (entry?.from.equals(publicKey)) {
-            this._resultsCollectQueue.del(query.idString);
-        }
-        else if (entry) {
-            logger.warn("Received a close iterator request for a iterator that does not belong to the requesting peer");
-        }
-    }
-    async handleFieldQuery(f, obj, startIndex) {
-        // this clause is needed if we have a field that is of type [][] (we will recursively go through each subarray)
-        if (Array.isArray(obj)) {
-            for (const element of obj) {
-                if (await this.handleFieldQuery(f, element, startIndex)) {
-                    return true;
-                }
-            }
-            return false;
-        }
-        // Resolve the field from the key path. If we reach an array or nested Document store,
-        // then do a recursive call or a search to look into them
-        for (let i = startIndex; i < f.key.length; i++) {
-            obj = obj[f.key[i]];
-            if (Array.isArray(obj)) {
-                for (const element of obj) {
-                    if (await this.handleFieldQuery(f, element, i + 1)) {
-                        return true;
-                    }
-                }
-                return false;
-            }
-            if (obj instanceof this.dbType) {
-                const queryCloned = f.clone();
-                queryCloned.key.splice(0, i + 1); // remove key path until the document store
-                const results = await obj.index.search(new SearchRequest({ query: [queryCloned] }));
-                return results.length > 0 ? true : false; // TODO return INNER HITS?
-            }
-        }
-        //  When we reach here, the field value (obj) is comparable
-        if (f instanceof StringMatch) {
-            let compare = f.value;
-            if (f.caseInsensitive) {
-                compare = compare.toLowerCase();
-            }
-            if (this.handleStringMatch(f, compare, obj)) {
-                return true;
-            }
-            return false;
-        }
-        else if (f instanceof ByteMatchQuery) {
-            if (obj instanceof Uint8Array === false) {
-                if (stringArraysEquals(f.key, this._indexByArr)) {
-                    return f.valueString === obj;
-                }
-                return false;
-            }
-            return equals(obj, f.value);
-        }
-        else if (f instanceof IntegerCompare) {
-            const value = obj;
-            if (typeof value !== "bigint" && typeof value !== "number") {
-                return false;
-            }
-            return compare(value, f.compare, f.value.value);
-        }
-        else if (f instanceof MissingField) {
-            return obj == null; // null or undefined
-        }
-        else if (f instanceof BoolQuery) {
-            return obj === f.value; // true/false
-        }
-        logger.warn("Unsupported query type: " + f.constructor.name);
-        return false;
-    }
-    async handleQueryObject(f, doc) {
-        if (f instanceof StateFieldQuery) {
-            return this.handleFieldQuery(f, doc.value, 0);
-        }
-        else if (f instanceof LogicalQuery) {
-            if (f instanceof And) {
-                for (const and of f.and) {
-                    if (!(await this.handleQueryObject(and, doc))) {
-                        return false;
-                    }
-                }
-                return true;
-            }
-            if (f instanceof Or) {
-                for (const or of f.or) {
-                    if (await this.handleQueryObject(or, doc)) {
-                        return true;
-                    }
-                }
-                return false;
-            }
-            return false;
-        }
-        logger.info("Unsupported query type: " + f.constructor.name);
-        return false;
-    }
-    handleStringMatch(f, compare, fv) {
-        if (typeof fv !== "string") {
-            return false;
-        }
-        if (f.caseInsensitive) {
-            fv = fv.toLowerCase();
-        }
-        if (f.method === StringMatchMethod.exact) {
-            return fv === compare;
-        }
-        if (f.method === StringMatchMethod.prefix) {
-            return fv.startsWith(compare);
-        }
-        if (f.method === StringMatchMethod.contains) {
-            return fv.includes(compare);
-        }
-        throw new Error("Unsupported");
-    }
-    /**
-     * Query and retrieve results with most details
-     * @param queryRequest
-     * @param options
-     * @returns
-     */
-    async queryDetailed(queryRequest, options) {
-        const local = typeof options?.local == "boolean" ? options?.local : true;
-        let remote = undefined;
-        if (typeof options?.remote === "boolean") {
-            if (options?.remote) {
-                remote = {};
-            }
-            else {
-                remote = undefined;
-            }
-        }
-        else {
-            remote = options?.remote || {};
-        }
-        if (remote && remote.priority == null) {
-            // give queries higher priority than other "normal" data activities
-            // without this, we might have a scenario that a peer joina  network with large amount of data to be synced, but can not query anything before that is done
-            // this will lead to bad UX as you usually want to list/expore whats going on before doing any replication work
-            remote.priority = 1;
-        }
-        const promises = [];
-        if (!local && !remote) {
-            throw new Error("Expecting either 'options.remote' or 'options.local' to be true");
-        }
-        const allResults = [];
-        if (local) {
-            const results = await this.processFetchRequest(queryRequest, this.node.identity.publicKey);
-            if (results.results.length > 0) {
-                const resultsObject = new Results({
-                    results: results.results.map((r) => {
-                        if (r.value.last instanceof PutOperation === false) {
-                            throw new Error("Unexpected value type on local results: " +
-                                r.value.last?.constructor.name || typeof r.value.last);
-                        }
-                        return new ResultWithSource({
-                            context: r.context,
-                            value: r.value.value,
-                            source: r.value.last.data
-                        });
-                    }),
-                    kept: BigInt(results.kept)
-                });
-                options?.onResponse &&
-                    options.onResponse(resultsObject, this.node.identity.publicKey);
-                allResults.push(resultsObject);
-            }
-        }
-        if (remote) {
-            const replicatorGroups = await this._log.getReplicatorUnion(remote.minAge);
-            if (replicatorGroups) {
-                const groupHashes = replicatorGroups.map((x) => [x]);
-                const fn = async () => {
-                    const rs = [];
-                    const responseHandler = async (results) => {
-                        for (const r of await introduceEntries(results, this.type, this._sync, options)) {
-                            rs.push(r.response);
-                        }
-                    };
-                    try {
-                        if (queryRequest instanceof CloseIteratorRequest) {
-                            // don't wait for responses
-                            await this._query.request(queryRequest, { mode: remote.mode });
-                        }
-                        else {
-                            await queryAll(this._query, groupHashes, queryRequest, responseHandler, remote);
-                        }
-                    }
-                    catch (error) {
-                        if (error instanceof MissingResponsesError) {
-                            logger.warn("Did not reciveve responses from all shard");
-                            if (remote?.throwOnMissing) {
-                                throw error;
-                            }
-                        }
-                        else {
-                            throw error;
-                        }
-                    }
-                    return rs;
-                };
-                promises.push(fn());
-            }
-            else {
-                // TODO send without direction out to the world? or just assume we can insert?
-                /* 	promises.push(
-                        this._query
-                            .request(queryRequest, remote)
-                            .then((results) => introduceEntries(results, this.type, this._sync, options).then(x => x.map(y => y.response)))
-                    ); */
-                /* throw new Error(
-                    "Missing remote replicator info for performing distributed document query"
-                ); */
-            }
-        }
-        const resolved = await Promise.all(promises);
-        for (const r of resolved) {
-            if (r) {
-                if (r instanceof Array) {
-                    allResults.push(...r);
-                }
-                else {
-                    allResults.push(r);
-                }
-            }
-        }
-        return allResults;
-    }
-    /**
-     * Query and retrieve results
-     * @param queryRequest
-     * @param options
-     * @returns
-     */
-    async search(queryRequest, options) {
-        // Set fetch to search size, or max value (default to max u32 (4294967295))
-        queryRequest.fetch = options?.size ?? 0xffffffff;
-        // So that the iterator is pre-fetching the right amount of entries
-        const iterator = this.iterate(queryRequest, options);
-        // So that this call will not do any remote requests
-        const allResults = [];
-        while (iterator.done() === false &&
-            queryRequest.fetch > allResults.length) {
-            // We might need to pull .next multiple time due to data message size limitations
-            for (const result of await iterator.next(queryRequest.fetch - allResults.length)) {
-                allResults.push(result);
-            }
-        }
-        await iterator.close();
-        //s Deduplicate and return values directly
-        return dedup(allResults, this.indexByResolver);
-    }
-    /**
-     * Query and retrieve documents in a iterator
-     * @param queryRequest
-     * @param options
-     * @returns
-     */
-    iterate(queryRequest, options) {
-        let fetchPromise = undefined;
-        const peerBufferMap = new Map();
-        const visited = new Set();
-        let done = false;
-        let first = false;
-        // TODO handle join/leave while iterating
-        const controller = new AbortController();
-        const peerBuffers = () => {
-            return [...peerBufferMap.values()].map((x) => x.buffer).flat();
-        };
-        const fetchFirst = async (n) => {
-            done = true; // Assume we are donne
-            queryRequest.fetch = n;
-            await this.queryDetailed(queryRequest, {
-                ...options,
-                onResponse: (response, from) => {
-                    if (!from) {
-                        logger.error("Missing response from");
-                        return;
-                    }
-                    if (response instanceof NoAccess) {
-                        logger.error("Dont have access");
-                        return;
-                    }
-                    else if (response instanceof Results) {
-                        const results = response;
-                        if (results.kept === 0n && results.results.length === 0) {
-                            return;
-                        }
-                        if (results.kept > 0n) {
-                            done = false; // we have more to do later!
-                        }
-                        peerBufferMap.set(from.hashcode(), {
-                            buffer: results.results
-                                .filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
-                                .map((x) => {
-                                visited.add(asString(this.indexByResolver(x.value)));
-                                return {
-                                    from,
-                                    value: { value: x.value },
-                                    context: x.context
-                                };
-                            }),
-                            kept: Number(response.kept)
-                        });
-                    }
-                    else {
-                        throw new Error("Unsupported result type: " + response?.constructor?.name);
-                    }
-                }
-            });
-            return done;
-        };
-        const fetchAtLeast = async (n) => {
-            if (done && first) {
-                return;
-            }
-            await fetchPromise;
-            if (!first) {
-                first = true;
-                fetchPromise = fetchFirst(n);
-                return fetchPromise;
-            }
-            const promises = [];
-            let resultsLeft = 0;
-            for (const [peer, buffer] of peerBufferMap) {
-                if (buffer.buffer.length < n) {
-                    if (buffer.kept === 0) {
-                        if (peerBufferMap.get(peer)?.buffer.length === 0) {
-                            peerBufferMap.delete(peer); // No more results
-                        }
-                        continue;
-                    }
-                    // TODO buffer more than deleted?
-                    // TODO batch to multiple 'to's
-                    const collectRequest = new CollectNextRequest({
-                        id: queryRequest.id,
-                        amount: n - buffer.buffer.length
-                    });
-                    // Fetch locally?
-                    if (peer === this.node.identity.publicKey.hashcode()) {
-                        promises.push(this.processFetchRequest(collectRequest, this.node.identity.publicKey)
-                            .then((results) => {
-                            resultsLeft += results.kept;
-                            if (results.results.length === 0) {
-                                if (peerBufferMap.get(peer)?.buffer.length === 0) {
-                                    peerBufferMap.delete(peer); // No more results
-                                }
-                            }
-                            else {
-                                const peerBuffer = peerBufferMap.get(peer);
-                                if (!peerBuffer) {
-                                    return;
-                                }
-                                peerBuffer.kept = results.kept;
-                                peerBuffer.buffer.push(...results.results
-                                    .filter((x) => !visited.has(asString(this.indexByResolver(x.value.value))))
-                                    .map((x) => {
-                                    visited.add(asString(this.indexByResolver(x.value.value)));
-                                    return {
-                                        value: x.value,
-                                        context: x.context,
-                                        from: this.node.identity.publicKey
-                                    };
-                                }));
-                            }
-                        })
-                            .catch((e) => {
-                            logger.error("Failed to collect sorted results from self. " + e?.message);
-                            peerBufferMap.delete(peer);
-                        }));
-                    }
-                    else {
-                        // Fetch remotely
-                        promises.push(this._query
-                            .request(collectRequest, {
-                            ...options,
-                            signal: controller.signal,
-                            priority: 1,
-                            mode: new SilentDelivery({ to: [peer], redundancy: 1 })
-                        })
-                            .then((response) => introduceEntries(response, this.type, this._sync, options)
-                            .then((responses) => {
-                            responses.map((response) => {
-                                resultsLeft += Number(response.response.kept);
-                                if (!response.from) {
-                                    logger.error("Missing from for sorted query");
-                                    return;
-                                }
-                                if (response.response.results.length === 0) {
-                                    if (peerBufferMap.get(peer)?.buffer.length === 0) {
-                                        peerBufferMap.delete(peer); // No more results
-                                    }
-                                }
-                                else {
-                                    const peerBuffer = peerBufferMap.get(peer);
-                                    if (!peerBuffer) {
-                                        return;
-                                    }
-                                    peerBuffer.kept = Number(response.response.kept);
-                                    peerBuffer.buffer.push(...response.response.results
-                                        .filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
-                                        .map((x) => {
-                                        visited.add(asString(this.indexByResolver(x.value)));
-                                        return {
-                                            value: { value: x.value },
-                                            context: x.context,
-                                            from: response.from
-                                        };
-                                    }));
-                                }
-                            });
-                        })
-                            .catch((e) => {
-                            logger.error("Failed to collect sorted results from: " +
-                                peer +
-                                ". " +
-                                e?.message);
-                            peerBufferMap.delete(peer);
-                        })));
-                    }
-                }
-                else {
-                    resultsLeft += peerBufferMap.get(peer)?.kept || 0;
-                }
-            }
-            return (fetchPromise = Promise.all(promises).then(() => {
-                return resultsLeft === 0; // 0 results left to fetch and 0 pending results
-            }));
-        };
-        const next = async (n) => {
-            if (n < 0) {
-                throw new Error("Expecting to fetch a positive amount of element");
-            }
-            if (n === 0) {
-                return [];
-            }
-            // TODO everything below is not very optimized
-            const fetchedAll = await fetchAtLeast(n);
-            // get n next top entries, shift and pull more results
-            const results = await resolvedSort(peerBuffers(), this._toIndex, queryRequest.sort);
-            const pendingMoreResults = n < results.length;
-            const batch = results.splice(0, n);
-            for (const result of batch) {
-                const arr = peerBufferMap.get(result.from.hashcode());
-                if (!arr) {
-                    logger.error("Unexpected empty result buffer");
-                    continue;
-                }
-                const idx = arr.buffer.findIndex((x) => x == result);
-                if (idx >= 0) {
-                    arr.buffer.splice(idx, 1);
-                }
-            }
-            done = fetchedAll && !pendingMoreResults;
-            return dedup(batch.map((x) => x.value.value), this.indexByResolver);
-        };
-        const close = async () => {
-            controller.abort(new AbortError("Iterator closed"));
-            const closeRequest = new CloseIteratorRequest({ id: queryRequest.id });
-            const promises = [];
-            for (const [peer, buffer] of peerBufferMap) {
-                if (buffer.kept === 0) {
-                    peerBufferMap.delete(peer);
-                    continue;
-                }
-                // Fetch locally?
-                if (peer === this.node.identity.publicKey.hashcode()) {
-                    promises.push(this.processCloseIteratorRequest(closeRequest, this.node.identity.publicKey));
-                }
-                else {
-                    // Close remote
-                    promises.push(this._query.send(closeRequest, {
-                        ...options,
-                        mode: new SilentDelivery({ to: [peer], redundancy: 1 })
-                    }));
-                }
-            }
-            await Promise.all(promises);
-        };
-        return {
-            close,
-            next,
-            done: () => done
-        };
-    }
-};
-__decorate([
-    field({ type: RPC }),
-    __metadata("design:type", RPC)
-], DocumentIndex.prototype, "_query", void 0);
-DocumentIndex = __decorate([
-    variant("documents_index"),
-    __metadata("design:paramtypes", [Object])
-], DocumentIndex);
-export { DocumentIndex };
-//# sourceMappingURL=document-index.js.map