@peerbit/document 6.0.6 → 6.0.7-218a5bb

package/{lib/esm/document-index.js → dist/src/search.js}

@@ -7,69 +7,42 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-import { field, variant } from "@dao-xyz/borsh";
-import { asString } from "./utils.js";
-import { BORSH_ENCODING } from "@peerbit/log";
-import { equals } from "@peerbit/uint8arrays";
+import { field, serialize, variant } from "@dao-xyz/borsh";
+import { BORSH_ENCODING, Entry } from "@peerbit/log";
 import { Program } from "@peerbit/program";
-import { IntegerCompare, ByteMatchQuery, StringMatch, ResultWithSource, StateFieldQuery, compare, MissingField, StringMatchMethod, LogicalQuery, And, Or, BoolQuery, CollectNextRequest, AbstractSearchRequest, SearchRequest, SortDirection, CloseIteratorRequest, NoAccess, AbstractSearchResult } from "./query.js";
+import * as types from "@peerbit/document-interface";
 import { RPC, queryAll, MissingResponsesError } from "@peerbit/rpc";
-import { Results } from "./query.js";
 import { logger as loggerFn } from "@peerbit/logger";
-import { Cache } from "@peerbit/cache";
-import { sha256Base64Sync } from "@peerbit/crypto";
+import { PublicSignKey, sha256Base64Sync } from "@peerbit/crypto";
+import { SharedLog } from "@peerbit/shared-log";
 import { concat, fromString } from "uint8arrays";
 import { SilentDelivery } from "@peerbit/stream-interface";
 import { AbortError } from "@peerbit/time";
+import { Cache } from "@peerbit/cache";
+import { HashmapIndexEngine } from "@peerbit/document-index-simple";
+import { MAX_BATCH_SIZE } from "./constants.js";
 const logger = loggerFn({ module: "document-index" });
-const stringArraysEquals = (a, b) => {
-    if (a.length !== b.length) {
-        return false;
-    }
-    for (let i = 0; i < a.length; i++) {
-        if (a[i] !== b[i]) {
-            return false;
-        }
-    }
-    return true;
-};
-let Operation = class Operation {
+let Operation = class Operation /* <T> */ {
 };
 Operation = __decorate([
     variant(0)
 ], Operation);
 export { Operation };
 export const BORSH_ENCODING_OPERATION = BORSH_ENCODING(Operation);
-let PutOperation = class PutOperation extends Operation {
-    key;
+/**
+ * Put a complete document at a key
+ */
+let PutOperation = class PutOperation extends Operation /* <T> */ {
     data;
-    _value;
+    /* _value?: T; */
     constructor(props) {
         super();
         if (props) {
-            this.key = props.key;
             this.data = props.data;
-            this._value = props.value;
+            /* this._value = props.value; */
         }
     }
-    get value() {
-        if (!this._value) {
-            throw new Error("Value not decoded, invoke getValue(...) once");
-        }
-        return this._value;
-    }
-    getValue(encoding) {
-        if (this._value) {
-            return this._value;
-        }
-        this._value = encoding.decoder(this.data);
-        return this._value;
-    }
 };
-__decorate([
-    field({ type: "string" }),
-    __metadata("design:type", String)
-], PutOperation.prototype, "key", void 0);
 __decorate([
     field({ type: Uint8Array }),
     __metadata("design:type", Uint8Array)
@@ -92,80 +65,41 @@ export class PutAllOperation<T> extends Operation<T> {
     }
 }
  */
+/**
+ * Delete a document at a key
+ */
 let DeleteOperation = class DeleteOperation extends Operation {
     key;
     constructor(props) {
         super();
-        if (props) {
-            this.key = props.key;
-        }
+        this.key = props.key;
     }
 };
 __decorate([
-    field({ type: "string" }),
-    __metadata("design:type", String)
+    field({ type: types.IdKey }),
+    __metadata("design:type", types.IdKey)
 ], DeleteOperation.prototype, "key", void 0);
 DeleteOperation = __decorate([
     variant(2),
     __metadata("design:paramtypes", [Object])
 ], DeleteOperation);
 export { DeleteOperation };
-const sortCompare = (av, bv) => {
-    if (typeof av === "string" && typeof bv === "string") {
-        return av.localeCompare(bv);
-    }
-    if (av < bv) {
-        return -1;
-    }
-    else if (av > bv) {
-        return 1;
-    }
-    return 0;
-};
-const extractFieldValue = (doc, path) => {
-    for (let i = 0; i < path.length; i++) {
-        doc = doc[path[i]];
-    }
-    return doc;
-};
-const extractSortCompare = (a, b, sorts) => {
-    for (const sort of sorts) {
-        const av = extractFieldValue(a, sort.key);
-        const bv = extractFieldValue(b, sort.key);
-        const cmp = sortCompare(av, bv);
-        if (cmp != 0) {
-            if (sort.direction === SortDirection.ASC) {
-                return cmp;
-            }
-            else {
-                return -cmp;
-            }
-        }
-    }
-    return 0;
-};
-const resolvedSort = async (arr, index, sorts) => {
-    await Promise.all(arr.map(async (result) => (result[SORT_TMP_KEY] = await index(result.value.value, result.context))));
-    arr.sort((a, b) => extractSortCompare(a[SORT_TMP_KEY], b[SORT_TMP_KEY], sorts));
-    return arr;
-};
-const SORT_TMP_KEY = "__sort_ref";
 const introduceEntries = async (responses, type, sync, options) => {
     const results = [];
     for (const response of responses) {
         if (!response.from) {
             logger.error("Missing from for response");
         }
-        if (response.response instanceof Results) {
+        if (response.response instanceof types.Results) {
             response.response.results.forEach((r) => r.init(type));
             if (typeof options?.remote !== "boolean" && options?.remote?.sync) {
                 await sync(response.response);
             }
             options?.onResponse &&
-                options.onResponse(response.response, response.from); // TODO fix types
+                (await options.onResponse(response.response, response.from)); // TODO fix types
             results.push(response);
         }
-        else if (response.response instanceof NoAccess) {
+        else if (response.response instanceof types.NoAccess) {
             logger.error("Search resulted in access error");
         }
         else {
@@ -178,7 +112,7 @@ const dedup = (allResult, dedupBy) => {
     const unique = new Set();
     const dedup = [];
     for (const result of allResult) {
-        const key = asString(dedupBy(result));
+        const key = types.toIdeable(dedupBy(result));
         if (unique.has(key)) {
             continue;
         }
@@ -188,73 +122,61 @@ const dedup = (allResult, dedupBy) => {
     return dedup;
 };
 const DEFAULT_INDEX_BY = "id";
-/*
-if (!(await this.canRead(message.sender))) {
-    throw new AccessError();
-} */
-export const MAX_DOCUMENT_SIZE = 5e6;
-const getBatchFromResults = (results, wantedSize, maxSize = MAX_DOCUMENT_SIZE) => {
-    const batch = [];
-    let size = 0;
-    for (const result of results) {
-        batch.push(result);
-        size += result.value.last.data.length;
-        if (size > maxSize) {
-            break;
-        }
-        if (wantedSize <= batch.length) {
-            break;
-        }
-    }
-    results.splice(0, batch.length);
-    return batch;
-};
 let DocumentIndex = class DocumentIndex extends Program {
     _query;
+    engine;
     type;
     dbType;
     // Index key
-    _indexBy;
-    _indexByArr;
-    // Resolve doc value by index key
+    indexBy;
+    indexByArr;
     indexByResolver;
-    // Indexed (transforms an docuemnt into an obj with fields that ought to be indexed)
-    _toIndex;
+    // Transformation, indexer
+    fields;
     _valueEncoding;
     _sync;
-    _index;
-    _resultsCollectQueue;
     _log;
+    _resolverProgramCache;
+    _resolverCache;
+    _isProgramValues;
     constructor(properties) {
         super();
         this._query = properties?.query || new RPC();
     }
-    get index() {
-        return this._index;
-    }
     get valueEncoding() {
         return this._valueEncoding;
     }
-    get toIndex() {
-        return this._toIndex;
-    }
     async open(properties) {
-        this._index = new Map();
         this._log = properties.log;
         this.type = properties.type;
+        // if this.type is a class that extends Program we want to do special functionality
+        this._isProgramValues = this.type instanceof Program;
         this.dbType = properties.dbType;
         this._sync = properties.sync;
-        this._toIndex = properties.fields;
-        this._indexBy = properties.indexBy || DEFAULT_INDEX_BY;
-        this._indexByArr = Array.isArray(this._indexBy)
-            ? this._indexBy
-            : [this._indexBy];
+        this.fields = properties.fields;
+        this.indexBy = properties.indexBy || DEFAULT_INDEX_BY;
+        this.indexByArr = Array.isArray(this.indexBy)
+            ? this.indexBy
+            : [this.indexBy];
         this.indexByResolver =
-            typeof this._indexBy === "string"
-                ? (obj) => obj[this._indexBy]
-                : (obj) => extractFieldValue(obj, this._indexBy);
+            typeof this.indexBy === "string"
+                ? (obj) => obj[this.indexBy]
+                : (obj) => types.extractFieldValue(obj, this.indexBy);
         this._valueEncoding = BORSH_ENCODING(this.type);
-        this._resultsCollectQueue = new Cache({ max: 10000 }); // TODO choose limit better
+        if (this._isProgramValues) {
+            this._resolverProgramCache = new Map();
+        }
+        this._resolverCache = new Cache({ max: 1000 }); // TODO choose limit better (adaptive)
+        this.engine = properties.engine || new HashmapIndexEngine();
+        await this.engine.init({
+            indexBy: this.indexBy,
+            nested: {
+                match: (obj) => obj instanceof this.dbType,
+                query: async (obj, query) => obj.index.search(query)
+            },
+            maxBatchSize: MAX_BATCH_SIZE
+        });
+        await this.engine.start?.();
         await this._query.open({
             topic: sha256Base64Sync(concat([this._log.log.id, fromString("/document")])),
             responseHandler: async (query, ctx) => {
@@ -263,68 +185,130 @@ let DocumentIndex = class DocumentIndex extends Program {
                     return;
                 }
                 if (properties.canSearch &&
-                    (query instanceof SearchRequest ||
-                        query instanceof CollectNextRequest) &&
+                    (query instanceof types.SearchRequest ||
+                        query instanceof types.CollectNextRequest) &&
                     !(await properties.canSearch(query, ctx.from))) {
-                    return new NoAccess();
+                    return new types.NoAccess();
                 }
-                if (query instanceof CloseIteratorRequest) {
+                if (query instanceof types.CloseIteratorRequest) {
                     this.processCloseIteratorRequest(query, ctx.from);
                 }
                 else {
-                    const results = await this.processFetchRequest(query, ctx.from, {
+                    const results = await this.processQuery(query, ctx.from, {
                         canRead: properties.canRead
                     });
-                    return new Results({
+                    return new types.Results({
                         // Even if results might have length 0, respond, because then we now at least there are no matching results
-                        results: results.results.map((r) => {
-                            if (r.value.last instanceof PutOperation === false) {
-                                throw new Error("Unexpected value type on local results: " +
-                                    r.value.last?.constructor.name ||
-                                    typeof r.value.last);
-                            }
-                            return new ResultWithSource({
-                                source: r.value.last.data,
-                                context: r.context
-                            });
-                        }),
-                        kept: BigInt(results.kept)
+                        results: results.results,
+                        kept: results.kept
                     });
                 }
             },
-            responseType: AbstractSearchResult,
-            queryType: AbstractSearchRequest
+            responseType: types.AbstractSearchResult,
+            queryType: types.AbstractSearchRequest
         });
     }
+    async close(from) {
+        const closed = await super.close(from);
+        if (closed) {
+            await this.engine.stop?.();
+        }
+        return closed;
+    }
+    async drop(from) {
+        const closed = await super.drop(from);
+        if (closed) {
+            await this.engine.stop?.();
+        }
+        return closed;
+    }
     async get(key, options) {
-        return (await this.getDetailed(key, options))?.[0]?.results[0]?.value;
+        return (await this.getDetailed(key instanceof types.IdKey ? key : types.toId(key), options))?.[0]?.results[0]?.value;
+    }
+    async put(value, entry, id) {
+        const idString = id.primitive;
+        if (this._isProgramValues) {
+            this._resolverProgramCache.set(idString, value);
+        }
+        else {
+            this._resolverCache.add(idString, value);
+        }
+        const context = new types.Context({
+            created: (await this.engine.get(id))?.context.created ||
+                entry.meta.clock.timestamp.wallTime,
+            modified: entry.meta.clock.timestamp.wallTime,
+            head: entry.hash,
+            gid: entry.gid
+        });
+        const valueToIndex = await this.fields(value, context);
+        this.engine.put({
+            id,
+            indexed: valueToIndex,
+            context,
+            size: entry.payload.data.byteLength
+            /* reference:
+                valueToIndex === value || value instanceof Program
+                    ? { value }
+                    : undefined */
+        });
+    }
+    del(key) {
+        const keyObject = types.toId(key);
+        if (this._isProgramValues) {
+            this._resolverProgramCache.delete(key);
+        }
+        else {
+            this._resolverCache.del(key);
+        }
+        return this.engine.del(keyObject);
     }
     async getDetailed(key, options) {
         let results;
         if (key instanceof Uint8Array) {
-            results = await this.queryDetailed(new SearchRequest({
-                query: [new ByteMatchQuery({ key: this._indexByArr, value: key })]
-            }), options);
-        }
-        else {
-            const stringValue = asString(key);
-            results = await this.queryDetailed(new SearchRequest({
+            results = await this.queryDetailed(new types.SearchRequest({
                 query: [
-                    new StringMatch({
-                        key: this._indexByArr,
-                        value: stringValue
-                    })
+                    new types.ByteMatchQuery({ key: this.indexByArr, value: key })
                 ]
             }), options);
         }
+        else {
+            const indexableKey = types.toIdeable(key);
+            if (typeof indexableKey === "number" ||
+                typeof indexableKey === "bigint") {
+                results = await this.queryDetailed(new types.SearchRequest({
+                    query: [
+                        new types.IntegerCompare({
+                            key: this.indexByArr,
+                            compare: types.Compare.Equal,
+                            value: indexableKey
+                        })
+                    ]
+                }), options);
+            }
+            else {
+                results = await this.queryDetailed(new types.SearchRequest({
+                    query: [
+                        new types.StringMatch({
+                            key: this.indexByArr,
+                            value: indexableKey
+                        })
+                    ]
+                }), options);
+            }
+        }
         return results;
     }
-    get size() {
-        return this._index.size;
+    getSize() {
+        return this.engine.getSize();
     }
-    async getDocumentWithLastOperation(value) {
-        if (value.reference) {
-            return value.reference;
+    async resolveDocument(value) {
+        const cached = this._resolverCache.get(value.id.primitive) ||
+            this._resolverProgramCache?.get(value.id.primitive);
+        if (cached != null) {
+            return { value: cached };
+        }
+        if (value.indexed instanceof this.type) {
+            return { value: value.indexed };
         }
         const head = await await this._log.log.get(value.context.head);
         if (!head) {
@@ -334,230 +318,47 @@ let DocumentIndex = class DocumentIndex extends Program {
         const payloadValue = await head.getPayloadValue();
         if (payloadValue instanceof PutOperation) {
             return {
-                value: payloadValue.getValue(this.valueEncoding),
-                last: payloadValue
+                value: this.valueEncoding.decoder(payloadValue.data)
+                /* size: payloadValue.data.byteLength */
             };
         }
         throw new Error("Unexpected value type when getting document: " +
             payloadValue?.constructor?.name || typeof payloadValue);
     }
-    getDocument(value) {
-        return this.getDocumentWithLastOperation(value).then((r) => r?.value);
-    }
-    async _queryDocuments(filter) {
-        // Whether we return the full operation data or just the db value
-        const results = [];
-        for (const value of this._index.values()) {
-            if (await filter(value)) {
-                const topDoc = await this.getDocumentWithLastOperation(value);
-                topDoc &&
-                    results.push({
-                        context: value.context,
-                        value: topDoc
-                    });
-            }
-        }
-        return results;
-    }
-    async processFetchRequest(query, from, options) {
+    async processQuery(query, from, options) {
         // We do special case for querying the id as we can do it faster than iterating
-        if (query instanceof SearchRequest) {
-            // Special case querying ids
-            if (query.query.length === 1 &&
-                (query.query[0] instanceof ByteMatchQuery ||
-                    query.query[0] instanceof StringMatch) &&
-                stringArraysEquals(query.query[0].key, this._indexByArr)) {
-                const firstQuery = query.query[0];
-                if (firstQuery instanceof ByteMatchQuery) {
-                    const doc = this._index.get(asString(firstQuery.value));
-                    const topDoc = doc && (await this.getDocumentWithLastOperation(doc));
-                    return topDoc
-                        ? {
-                            results: [
-                                {
-                                    value: topDoc,
-                                    context: doc.context
-                                }
-                            ],
-                            kept: 0
-                        }
-                        : { results: [], kept: 0 };
-                }
-                else if (firstQuery instanceof StringMatch &&
-                    firstQuery.method === StringMatchMethod.exact &&
-                    firstQuery.caseInsensitive === false) {
-                    const doc = this._index.get(firstQuery.value);
-                    const topDoc = doc && (await this.getDocumentWithLastOperation(doc));
-                    return topDoc
-                        ? {
-                            results: [
-                                {
-                                    value: topDoc,
-                                    context: doc.context
-                                }
-                            ],
-                            kept: 0
-                        }
-                        : { results: [], kept: 0 };
-                }
-            }
-            // Handle query normally
-            let results = await this._queryDocuments(async (doc) => {
-                for (const f of query.query) {
-                    if (!(await this.handleQueryObject(f, doc))) {
-                        return false;
-                    }
-                }
-                return true;
-            });
-            if (options?.canRead) {
-                const keepFilter = await Promise.all(results.map((x) => options?.canRead(x.value.value, from)));
-                results = results.filter((x, i) => keepFilter[i]);
-            }
-            // Sort
-            await resolvedSort(results, this._toIndex, query.sort);
-            const batch = getBatchFromResults(results, query.fetch);
-            if (results.length > 0) {
-                this._resultsCollectQueue.add(query.idString, {
-                    arr: results,
-                    from
-                }); // cache resulst not returned
-            }
-            return { results: batch, kept: results.length }; // Only return 1 result since we are doing distributed sort, TODO buffer more initially
-        }
-        else if (query instanceof CollectNextRequest) {
-            const results = this._resultsCollectQueue.get(query.idString);
-            if (!results) {
-                return {
-                    results: [],
-                    kept: 0
-                };
-            }
-            const batch = getBatchFromResults(results.arr, query.amount);
-            if (results.arr.length === 0) {
-                this._resultsCollectQueue.del(query.idString); // TODO add tests for proper cleanup/timeouts
-            }
-            return { results: batch, kept: results.arr.length };
-        }
-        throw new Error("Unsupported");
-    }
-    async processCloseIteratorRequest(query, publicKey) {
-        const entry = this._resultsCollectQueue.get(query.idString);
-        if (entry?.from.equals(publicKey)) {
-            this._resultsCollectQueue.del(query.idString);
-        }
-        else if (entry) {
-            logger.warn("Received a close iterator request for a iterator that does not belong to the requesting peer");
-        }
-    }
-    async handleFieldQuery(f, obj, startIndex) {
-        // this clause is needed if we have a field that is of type [][] (we will recursively go through each subarray)
-        if (Array.isArray(obj)) {
-            for (const element of obj) {
-                if (await this.handleFieldQuery(f, element, startIndex)) {
-                    return true;
-                }
-            }
-            return false;
-        }
-        // Resolve the field from the key path. If we reach an array or nested Document store,
-        // then do a recursive call or a search to look into them
-        for (let i = startIndex; i < f.key.length; i++) {
-            obj = obj[f.key[i]];
-            if (Array.isArray(obj)) {
-                for (const element of obj) {
-                    if (await this.handleFieldQuery(f, element, i + 1)) {
-                        return true;
-                    }
-                }
-                return false;
-            }
-            if (obj instanceof this.dbType) {
-                const queryCloned = f.clone();
-                queryCloned.key.splice(0, i + 1); // remove key path until the document store
-                const results = await obj.index.search(new SearchRequest({ query: [queryCloned] }));
-                return results.length > 0 ? true : false; // TODO return INNER HITS?
-            }
+        let indexedResult = undefined;
+        if (query instanceof types.SearchRequest) {
+            indexedResult = await this.engine.query(query, from);
         }
-        //  When we reach here, the field value (obj) is comparable
-        if (f instanceof StringMatch) {
-            let compare = f.value;
-            if (f.caseInsensitive) {
-                compare = compare.toLowerCase();
-            }
-            if (this.handleStringMatch(f, compare, obj)) {
-                return true;
-            }
-            return false;
+        else if (query instanceof types.CollectNextRequest) {
+            indexedResult = await this.engine.next(query, from);
         }
-        else if (f instanceof ByteMatchQuery) {
-            if (obj instanceof Uint8Array === false) {
-                if (stringArraysEquals(f.key, this._indexByArr)) {
-                    return f.valueString === obj;
-                }
-                return false;
-            }
-            return equals(obj, f.value);
-        }
-        else if (f instanceof IntegerCompare) {
-            const value = obj;
-            if (typeof value !== "bigint" && typeof value !== "number") {
-                return false;
-            }
-            return compare(value, f.compare, f.value.value);
-        }
-        else if (f instanceof MissingField) {
-            return obj == null; // null or undefined
-        }
-        else if (f instanceof BoolQuery) {
-            return obj === f.value; // true/false
+        else {
+            throw new Error("Unsupported");
         }
-        logger.warn("Unsupported query type: " + f.constructor.name);
-        return false;
-    }
-    async handleQueryObject(f, doc) {
-        if (f instanceof StateFieldQuery) {
-            return this.handleFieldQuery(f, doc.value, 0);
-        }
-        else if (f instanceof LogicalQuery) {
-            if (f instanceof And) {
-                for (const and of f.and) {
-                    if (!(await this.handleQueryObject(and, doc))) {
-                        return false;
-                    }
-                }
-                return true;
-            }
-            if (f instanceof Or) {
-                for (const or of f.or) {
-                    if (await this.handleQueryObject(or, doc)) {
-                        return true;
-                    }
-                }
-                return false;
-            }
-            return false;
+        const filteredResults = [];
+        for (const result of indexedResult.results) {
+            const value = await this.resolveDocument(result);
+            if (!value ||
+                (options?.canRead && !(await options.canRead(value.value, from)))) {
+                continue;
+            }
+            filteredResults.push(new types.ResultWithSource({
+                context: result.context,
+                value: value.value,
+                source: serialize(value.value),
+                indexed: result.indexed
+            }));
         }
-        logger.info("Unsupported query type: " + f.constructor.name);
-        return false;
+        const results = new types.Results({
+            results: filteredResults,
+            kept: BigInt(indexedResult.kept)
+        });
+        return results;
     }
-    handleStringMatch(f, compare, fv) {
-        if (typeof fv !== "string") {
-            return false;
-        }
-        if (f.caseInsensitive) {
-            fv = fv.toLowerCase();
-        }
-        if (f.method === StringMatchMethod.exact) {
-            return fv === compare;
-        }
-        if (f.method === StringMatchMethod.prefix) {
-            return fv.startsWith(compare);
-        }
-        if (f.method === StringMatchMethod.contains) {
-            return fv.includes(compare);
-        }
-        throw new Error("Unsupported");
+    async processCloseIteratorRequest(query, publicKey) {
+        return this.engine.close(query, publicKey);
     }
     /**
      * Query and retrieve results with most details
@@ -591,25 +392,11 @@ let DocumentIndex = class DocumentIndex extends Program {
         }
         const allResults = [];
         if (local) {
-            const results = await this.processFetchRequest(queryRequest, this.node.identity.publicKey);
+            const results = await this.processQuery(queryRequest, this.node.identity.publicKey);
             if (results.results.length > 0) {
-                const resultsObject = new Results({
-                    results: results.results.map((r) => {
-                        if (r.value.last instanceof PutOperation === false) {
-                            throw new Error("Unexpected value type on local results: " +
-                                r.value.last?.constructor.name || typeof r.value.last);
-                        }
-                        return new ResultWithSource({
-                            context: r.context,
-                            value: r.value.value,
-                            source: r.value.last.data
-                        });
-                    }),
-                    kept: BigInt(results.kept)
-                });
                 options?.onResponse &&
-                    options.onResponse(resultsObject, this.node.identity.publicKey);
-                allResults.push(resultsObject);
+                    (await options.onResponse(results, this.node.identity.publicKey));
+                allResults.push(results);
             }
         }
         if (remote) {
@@ -624,7 +411,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                         }
                     };
                     try {
-                        if (queryRequest instanceof CloseIteratorRequest) {
+                        if (queryRequest instanceof types.CloseIteratorRequest) {
                             // don't wait for responses
                             await this._query.request(queryRequest, { mode: remote.mode });
                         }
@@ -718,16 +505,16 @@ let DocumentIndex = class DocumentIndex extends Program {
             queryRequest.fetch = n;
             await this.queryDetailed(queryRequest, {
                 ...options,
-                onResponse: (response, from) => {
+                onResponse: async (response, from) => {
                     if (!from) {
                         logger.error("Missing response from");
                         return;
                     }
-                    if (response instanceof NoAccess) {
+                    if (response instanceof types.NoAccess) {
                         logger.error("Dont have access");
                         return;
                     }
-                    else if (response instanceof Results) {
+                    else if (response instanceof types.Results) {
                         const results = response;
                         if (results.kept === 0n && results.results.length === 0) {
                             return;
@@ -735,17 +522,23 @@ let DocumentIndex = class DocumentIndex extends Program {
                         if (results.kept > 0n) {
                             done = false; // we have more to do later!
                         }
+                        const buffer = [];
+                        for (const result of results.results) {
+                            const indexKey = types.toIdeable(this.indexByResolver(result.value));
+                            if (visited.has(indexKey)) {
+                                continue;
+                            }
+                            visited.add(indexKey);
+                            buffer.push({
+                                value: result.value,
+                                context: result.context,
+                                from: from,
+                                indexed: result.indexed ||
+                                    (await this.fields(result.value, result.context))
+                            });
+                        }
                         peerBufferMap.set(from.hashcode(), {
-                            buffer: results.results
-                                .filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
-                                .map((x) => {
-                                visited.add(asString(this.indexByResolver(x.value)));
-                                return {
-                                    from,
-                                    value: { value: x.value },
-                                    context: x.context
-                                };
-                            }),
+                            buffer,
                             kept: Number(response.kept)
                         });
                     }
@@ -778,15 +571,15 @@ let DocumentIndex = class DocumentIndex extends Program {
                     }
                     // TODO buffer more than deleted?
                     // TODO batch to multiple 'to's
-                    const collectRequest = new CollectNextRequest({
+                    const collectRequest = new types.CollectNextRequest({
                         id: queryRequest.id,
                         amount: n - buffer.buffer.length
                     });
                     // Fetch locally?
                     if (peer === this.node.identity.publicKey.hashcode()) {
-                        promises.push(this.processFetchRequest(collectRequest, this.node.identity.publicKey)
-                            .then((results) => {
-                            resultsLeft += results.kept;
+                        promises.push(this.processQuery(collectRequest, this.node.identity.publicKey)
+                            .then(async (results) => {
+                            resultsLeft += Number(results.kept);
                             if (results.results.length === 0) {
                                 if (peerBufferMap.get(peer)?.buffer.length === 0) {
                                     peerBufferMap.delete(peer); // No more results
@@ -797,17 +590,20 @@ let DocumentIndex = class DocumentIndex extends Program {
                                 if (!peerBuffer) {
                                     return;
                                 }
-                                peerBuffer.kept = results.kept;
-                                peerBuffer.buffer.push(...results.results
-                                    .filter((x) => !visited.has(asString(this.indexByResolver(x.value.value))))
-                                    .map((x) => {
-                                    visited.add(asString(this.indexByResolver(x.value.value)));
-                                    return {
-                                        value: x.value,
-                                        context: x.context,
-                                        from: this.node.identity.publicKey
-                                    };
-                                }));
+                                peerBuffer.kept = Number(results.kept);
+                                for (const result of results.results) {
+                                    if (visited.has(types.toIdeable(this.indexByResolver(result.value)))) {
+                                        continue;
+                                    }
+                                    visited.add(types.toIdeable(this.indexByResolver(result.value)));
+                                    peerBuffer.buffer.push({
+                                        value: result.value,
+                                        context: result.context,
+                                        from: this.node.identity.publicKey,
+                                        indexed: result.indexed ||
+                                            (await this.fields(result.value, result.context))
+                                    });
+                                }
                             }
                         })
                             .catch((e) => {
@@ -843,16 +639,18 @@ let DocumentIndex = class DocumentIndex extends Program {
                                         return;
                                     }
                                     peerBuffer.kept = Number(response.response.kept);
-                                    peerBuffer.buffer.push(...response.response.results
-                                        .filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
-                                        .map((x) => {
-                                        visited.add(asString(this.indexByResolver(x.value)));
-                                        return {
-                                            value: { value: x.value },
-                                            context: x.context,
-                                            from: response.from
-                                        };
-                                    }));
+                                    for (const result of response.response.results) {
+                                        if (visited.has(types.toIdeable(this.indexByResolver(result.value)))) {
+                                            continue;
+                                        }
+                                        visited.add(types.toIdeable(this.indexByResolver(result.value)));
+                                        peerBuffer.buffer.push({
+                                            value: result.value,
+                                            context: result.context,
+                                            from: response.from,
+                                            indexed: this.fields(result.value, result.context)
+                                        });
+                                    }
                                 }
                             });
                         })
@@ -883,7 +681,7 @@ let DocumentIndex = class DocumentIndex extends Program {
             // TODO everything below is not very optimized
             const fetchedAll = await fetchAtLeast(n);
             // get n next top entries, shift and pull more results
-            const results = await resolvedSort(peerBuffers(), this._toIndex, queryRequest.sort);
+            const results = await types.resolvedSort(peerBuffers(), queryRequest.sort);
             const pendingMoreResults = n < results.length;
             const batch = results.splice(0, n);
             for (const result of batch) {
@@ -892,17 +690,19 @@ let DocumentIndex = class DocumentIndex extends Program {
                     logger.error("Unexpected empty result buffer");
                     continue;
                 }
-                const idx = arr.buffer.findIndex((x) => x == result);
+                const idx = arr.buffer.findIndex((x) => x.value == result.value);
                 if (idx >= 0) {
                     arr.buffer.splice(idx, 1);
                 }
             }
             done = fetchedAll && !pendingMoreResults;
-            return dedup(batch.map((x) => x.value.value), this.indexByResolver);
+            return dedup(batch.map((x) => x.value), this.indexByResolver);
         };
         const close = async () => {
             controller.abort(new AbortError("Iterator closed"));
-            const closeRequest = new CloseIteratorRequest({ id: queryRequest.id });
+            const closeRequest = new types.CloseIteratorRequest({
+                id: queryRequest.id
+            });
             const promises = [];
             for (const [peer, buffer] of peerBufferMap) {
                 if (buffer.kept === 0) {
@@ -939,4 +739,4 @@ DocumentIndex = __decorate([
     __metadata("design:paramtypes", [Object])
 ], DocumentIndex);
 export { DocumentIndex };
-//# sourceMappingURL=document-index.js.map
+//# sourceMappingURL=search.js.map