@peerbit/document 8.2.0 → 9.0.0

package/dist/src/search.js

@@ -8,9 +8,8 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 import { field, serialize, variant } from "@dao-xyz/borsh";
-import { BlockResponse } from "@peerbit/blocks";
 import { Cache } from "@peerbit/cache";
-import { PublicSignKey, sha256Base64Sync, } from "@peerbit/crypto";
+import { PublicSignKey, getPublicKeyFromPeerId, sha256Base64Sync, } from "@peerbit/crypto";
 import * as types from "@peerbit/document-interface";
 import * as indexerTypes from "@peerbit/indexer-interface";
 import { HashmapIndex } from "@peerbit/indexer-simple";
@@ -18,23 +17,48 @@ import { BORSH_ENCODING, Entry } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import { ClosedError, Program } from "@peerbit/program";
 import { MissingResponsesError, RPC, queryAll, } from "@peerbit/rpc";
-import { BlocksMessage, SharedLog, } from "@peerbit/shared-log";
+import { SharedLog } from "@peerbit/shared-log";
 import { SilentDelivery } from "@peerbit/stream-interface";
-import { AbortError } from "@peerbit/time";
+import { AbortError, waitFor } from "@peerbit/time";
 import { concat, fromString } from "uint8arrays";
 import { copySerialization } from "./borsh.js";
 import { MAX_BATCH_SIZE } from "./constants.js";
 import { isPutOperation } from "./operation.js";
 import { ResumableIterators } from "./resumable-iterator.js";
 const logger = loggerFn({ module: "document-index" });
-const introduceEntries = async (queryRequest, responses, type, sync, options) => {
+const coerceQuery = (query, options) => {
+    let replicate = typeof options?.remote !== "boolean" ? options?.remote?.replicate : false;
+    if (query instanceof types.SearchRequestIndexed &&
+        query.replicate === false &&
+        replicate) {
+        query.replicate = true;
+        return query;
+    }
+    if (query instanceof types.SearchRequest) {
+        return query;
+    }
+    const queryObject = query;
+    return options?.resolve || options?.resolve == null
+        ? new types.SearchRequest({
+            query: indexerTypes.toQuery(queryObject.query),
+            sort: indexerTypes.toSort(query.sort),
+        })
+        : new types.SearchRequestIndexed({
+            query: indexerTypes.toQuery(queryObject.query),
+            sort: indexerTypes.toSort(query.sort),
+            replicate,
+        });
+};
+const introduceEntries = async (queryRequest, responses, documentType, indexedType, sync, options) => {
     const results = [];
     for (const response of responses) {
         if (!response.from) {
             logger.error("Missing from for response");
         }
         if (response.response instanceof types.Results) {
-            response.response.results.forEach((r) => r.init(type));
+            response.response.results.forEach((r) => r instanceof types.ResultValue
+                ? r.init(documentType)
+                : r.init(indexedType));
             if (typeof options?.remote !== "boolean" && options?.remote?.replicate) {
                 await sync(queryRequest, response.response);
             }
@@ -124,6 +148,7 @@ let DocumentIndex = class DocumentIndex extends Program {
     transformer;
     // The indexed document wrapped in a context
     wrappedIndexedType;
+    indexedType;
     // The database type, for recursive indexing
     dbType;
     indexedTypeIsDocumentType;
@@ -132,7 +157,7 @@ let DocumentIndex = class DocumentIndex extends Program {
     indexByResolver;
     index;
     _resumableIterators;
-    emitBlocksEagerly;
+    compatibility;
     // Transformation, indexer
     /* fields: IndexableFields<T, I>; */
     _valueEncoding;
@@ -155,7 +180,7 @@ let DocumentIndex = class DocumentIndex extends Program {
         this.indexedTypeIsDocumentType =
             !properties.transform?.type ||
                 properties.transform?.type === properties.documentType;
-        this.emitBlocksEagerly = properties.emitBlocksEagerly || false;
+        this.compatibility = properties.compatibility;
         let IndexedClassWithContext = class IndexedClassWithContext {
             __context;
             constructor(value, context) {
@@ -172,13 +197,62 @@ let DocumentIndex = class DocumentIndex extends Program {
             __metadata("design:paramtypes", [Object, IndexableContext])
         ], IndexedClassWithContext);
         // copy all prototype values from indexedType to IndexedClassWithContext
-        copySerialization((properties.transform?.type || properties.documentType), IndexedClassWithContext);
+        this.indexedType = (properties.transform?.type || properties.documentType);
+        copySerialization(this.indexedType, IndexedClassWithContext);
         this.wrappedIndexedType = IndexedClassWithContext;
         // if this.type is a class that extends Program we want to do special functionality
         this._isProgramValues = this.documentType instanceof Program;
         this.dbType = properties.dbType;
         this._resultQueue = new Map();
-        this._sync = properties.sync;
+        this._sync = async (request, results) => {
+            /*
+            let allPromises: Promise<void> | undefined = undefined
+                if (waitForValue) {
+                    let promises: Map<string, DeferredPromise<T>> = new Map();
+    
+                for (const result of results) {
+                    for (let i = 0; i < result.results.length; i++) {
+                        let promise = defer<T>();
+                        let r = result.results[i];
+                            promises.set(r.context.head, promise);
+                        const head = result.results[0].context.head;
+                        let listeners = this.hashToValueListener.get(head);
+                        if (!listeners) {
+                            listeners = [];
+                            this.hashToValueListener.set(head, listeners);
+                        }
+                        listeners.push(async (value) => {
+                                promise.resolve(value);
+                            result.results[i] = new types.ResultValue<T>({
+                                context: r.context,
+                                value,
+                                source: serialize(value),
+                                indexed: r.indexed,
+                            }) as any;
+                        });
+                        promise.promise.finally(() => {
+                            this.hashToValueListener.delete(head);
+                        });
+                    }
+                }
+
+                let timeout = setTimeout(() => {
+                    for (const promise of promises!) {
+                        promise[1].reject("Timed out resolving search result from value");
+                    }
+                }, 1e4);
+
+                allPromises = Promise.all([...promises.values()].map((x) => x.promise)).then(
+                    () => {
+                        clearTimeout(timeout);
+                    },
+                );
+            } */
+            await properties.replicate(request, results);
+            /* if (allPromises) {
+                await allPromises;
+            } */
+        };
         const transformOptions = properties.transform;
         this.transformer = transformOptions
             ? isTransformerWithFunction(transformOptions)
@@ -226,22 +300,6 @@ let DocumentIndex = class DocumentIndex extends Program {
                     const results = await this.processQuery(query, ctx.from, false, {
                         canRead: properties.canRead,
                     });
-                    if (this.emitBlocksEagerly) {
-                        await Promise.all(results.results.map(async (x) => {
-                            const hash = x.context.head;
-                            const block = await this._log.log.blocks.get(hash);
-                            if (!block) {
-                                return;
-                            }
-                            // todo dont do bytes -> entry -> bytes, but send the block directly
-                            return this._log.rpc.send(new BlocksMessage(new BlockResponse(hash, block)), {
-                                mode: new SilentDelivery({
-                                    to: [ctx.from],
-                                    redundancy: 1,
-                                }),
-                            });
-                        }));
-                    }
                     return new types.Results({
                         // Even if results might have length 0, respond, because then we now at least there are no matching results
                         results: results.results,
@@ -320,8 +378,12 @@ let DocumentIndex = class DocumentIndex extends Program {
     }
     async getDetailed(key, options) {
         let results;
+        const resolve = options?.resolve || options?.resolve == null;
+        let requestClazz = resolve
+            ? types.SearchRequest
+            : types.SearchRequestIndexed;
         if (key instanceof Uint8Array) {
-            results = await this.queryDetailed(new types.SearchRequest({
+            results = await this.queryCommence(new requestClazz({
                 query: [
                     new indexerTypes.ByteMatchQuery({ key: this.indexBy, value: key }),
                 ],
@@ -331,7 +393,7 @@ let DocumentIndex = class DocumentIndex extends Program {
             const indexableKey = indexerTypes.toIdeable(key);
             if (typeof indexableKey === "number" ||
                 typeof indexableKey === "bigint") {
-                results = await this.queryDetailed(new types.SearchRequest({
+                results = await this.queryCommence(new requestClazz({
                     query: [
                         new indexerTypes.IntegerCompare({
                             key: this.indexBy,
@@ -342,7 +404,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                 }), options);
             }
             else if (typeof indexableKey === "string") {
-                results = await this.queryDetailed(new types.SearchRequest({
+                results = await this.queryCommence(new requestClazz({
                     query: [
                         new indexerTypes.StringMatch({
                             key: this.indexBy,
@@ -352,7 +414,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                 }), options);
             }
             else if (indexableKey instanceof Uint8Array) {
-                results = await this.queryDetailed(new types.SearchRequest({
+                results = await this.queryCommence(new requestClazz({
                     query: [
                         new indexerTypes.ByteMatchQuery({
                             key: this.indexBy,
@@ -362,24 +424,48 @@ let DocumentIndex = class DocumentIndex extends Program {
                 }), options);
             }
         }
+        if (resolve &&
+            requestClazz === types.SearchRequestIndexed &&
+            !this.indexedTypeIsDocumentType &&
+            results) {
+            for (const set of results) {
+                let coercedResult = [];
+                for (const value of set.results) {
+                    const resolved = value instanceof types.ResultIndexedValue
+                        ? (await this.resolveDocument({
+                            indexed: value.value,
+                            head: value.context.head,
+                        }))?.value
+                        : value.value;
+                    if (resolved) {
+                        coercedResult.push(new types.ResultValue({
+                            context: value.context,
+                            value: resolved,
+                        }));
+                    }
+                }
+                set.results = coercedResult;
+            }
+        }
         return results;
     }
     getSize() {
         return this.index.getSize();
     }
     async resolveDocument(value) {
-        const cached = this._resolverCache.get(value.id.primitive) ||
-            this._resolverProgramCache?.get(value.id.primitive);
+        const id = value.id ??
+            indexerTypes.toId(this.indexByResolver(value.indexed)).primitive;
+        const cached = this._resolverCache.get(id) || this._resolverProgramCache?.get(id);
         if (cached != null) {
             return { value: cached };
         }
         if (this.indexedTypeIsDocumentType) {
             // cast value to T, i.e. convert the class but keep all properties except the __context
-            const obj = Object.assign(Object.create(this.documentType.prototype), value.value);
+            const obj = Object.assign(Object.create(this.documentType.prototype), value.indexed);
             delete obj.__context;
             return { value: obj };
         }
-        const head = await this._log.log.get(value.value.__context.head);
+        const head = await this._log.log.get(value.head);
         if (!head) {
             return undefined; // we could end up here if we recently pruned the document and other peers never persisted the entry
             // TODO update changes in index before removing entries from log entry storage
@@ -403,10 +489,14 @@ let DocumentIndex = class DocumentIndex extends Program {
             throw new Error("Different from in queued results");
         }
         let indexedResult = undefined;
-        if (query instanceof types.SearchRequest) {
+        let fromQuery;
+        if (query instanceof types.SearchRequest ||
+            query instanceof types.SearchRequestIndexed) {
+            fromQuery = query;
             indexedResult = await this._resumableIterators.iterateAndFetch(query);
         }
         else if (query instanceof types.CollectNextRequest) {
+            fromQuery = this._resumableIterators.queues.get(query.idString)?.request;
             indexedResult =
                 prevQueued?.keptInIndex === 0
                     ? []
@@ -415,7 +505,6 @@ let DocumentIndex = class DocumentIndex extends Program {
         else {
             throw new Error("Unsupported");
         }
-        const filteredResults = [];
         let resultSize = 0;
         let toIterate = prevQueued
             ? [...prevQueued.queue, ...indexedResult]
@@ -437,6 +526,7 @@ let DocumentIndex = class DocumentIndex extends Program {
             };
             this._resultQueue.set(query.idString, prevQueued);
         }
+        const filteredResults = [];
         for (const result of toIterate) {
             if (!isLocal) {
                 resultSize += result.value.__context.size;
@@ -445,17 +535,40 @@ let DocumentIndex = class DocumentIndex extends Program {
                     continue;
                 }
             }
-            const value = await this.resolveDocument(result);
-            if (!value ||
-                (options?.canRead && !(await options.canRead(value.value, from)))) {
+            const indexedUnwrapped = Object.assign(Object.create(this.indexedType.prototype), result.value);
+            if (options?.canRead &&
+                !(await options.canRead(indexedUnwrapped, from))) {
                 continue;
             }
-            filteredResults.push(new types.ResultWithSource({
-                context: result.value.__context.toContext(),
-                value: value.value,
-                source: serialize(value.value),
-                indexed: result.value,
-            }));
+            if (fromQuery instanceof types.SearchRequest) {
+                const value = await this.resolveDocument({
+                    indexed: result.value,
+                    head: result.value.__context.head,
+                });
+                if (!value) {
+                    continue;
+                }
+                filteredResults.push(new types.ResultValue({
+                    context: result.value.__context.toContext(),
+                    value: value.value,
+                    source: serialize(value.value),
+                    indexed: indexedUnwrapped,
+                }));
+            }
+            else if (fromQuery instanceof types.SearchRequestIndexed) {
+                const context = result.value.__context.toContext();
+                const head = await this._log.log.get(context.head);
+                // assume remote peer will start to replicate (TODO is this ideal?)
+                if (fromQuery.replicate) {
+                    this._log.addPeersToGidPeerHistory(context.gid, [from.hashcode()]);
+                }
+                filteredResults.push(new types.ResultIndexedValue({
+                    context,
+                    source: serialize(indexedUnwrapped),
+                    indexed: indexedUnwrapped,
+                    entries: head ? [head] : [],
+                }));
+            }
         }
         const results = new types.Results({
             results: filteredResults,
@@ -495,7 +608,7 @@ let DocumentIndex = class DocumentIndex extends Program {
      * @param options
      * @returns
      */
-    async queryDetailed(queryRequest, options) {
+    async queryCommence(queryRequest, options) {
         const local = typeof options?.local === "boolean" ? options?.local : true;
         let remote = undefined;
         if (typeof options?.remote === "boolean") {
@@ -536,7 +649,8 @@ let DocumentIndex = class DocumentIndex extends Program {
             if (replicatorGroups) {
                 const groupHashes = replicatorGroups.map((x) => [x]);
                 const responseHandler = async (results) => {
-                    for (const r of await introduceEntries(queryRequest, results, this.documentType, this._sync, options)) {
+                    const resultInitialized = await introduceEntries(queryRequest, results, this.documentType, this.indexedType, this._sync, options);
+                    for (const r of resultInitialized) {
                         resolved.push(r.response);
                     }
                 };
@@ -583,7 +697,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                 }
             }
         }
-        return allResults;
+        return allResults; // TODO types
     }
     /**
      * Query and retrieve results
@@ -593,19 +707,21 @@ let DocumentIndex = class DocumentIndex extends Program {
      */
     async search(queryRequest, options) {
         // Set fetch to search size, or max value (default to max u32 (4294967295))
-        queryRequest.fetch = queryRequest.fetch ?? 0xffffffff;
+        const coercedRequest = coerceQuery(queryRequest, options);
+        coercedRequest.fetch = coercedRequest.fetch ?? 0xffffffff;
         // So that the iterator is pre-fetching the right amount of entries
-        const iterator = this.iterate(queryRequest, options);
+        const iterator = this.iterate(coercedRequest, options);
         // So that this call will not do any remote requests
         const allResults = [];
-        while (iterator.done() !== true && queryRequest.fetch > allResults.length) {
+        while (iterator.done() !== true &&
+            coercedRequest.fetch > allResults.length) {
             // We might need to pull .next multiple time due to data message size limitations
-            for (const result of await iterator.next(queryRequest.fetch - allResults.length)) {
+            for (const result of await iterator.next(coercedRequest.fetch - allResults.length)) {
                 allResults.push(result);
             }
         }
         await iterator.close();
-        //s Deduplicate and return values directly
+        // Deduplicate and return values directly
         return dedup(allResults, this.indexByResolver);
     }
     /**
@@ -615,6 +731,30 @@ let DocumentIndex = class DocumentIndex extends Program {
      * @returns
      */
     iterate(queryRequest, options) {
+        let queryRequestCoerced = coerceQuery(queryRequest, options);
+        let resolve = false;
+        if (options?.remote &&
+            typeof options.remote !== "boolean" &&
+            options.remote.replicate &&
+            options?.resolve !== false) {
+            if ((queryRequest instanceof types.SearchRequestIndexed === false &&
+                this.compatibility == null) ||
+                (this.compatibility != null && this.compatibility > 8)) {
+                queryRequestCoerced = new types.SearchRequestIndexed({
+                    query: queryRequestCoerced.query,
+                    fetch: queryRequestCoerced.fetch,
+                    sort: queryRequestCoerced.sort,
+                });
+                resolve = true;
+            }
+        }
+        let replicate = options?.remote &&
+            typeof options.remote !== "boolean" &&
+            options.remote.replicate;
+        if (replicate &&
+            queryRequestCoerced instanceof types.SearchRequestIndexed) {
+            queryRequestCoerced.replicate = true;
+        }
         let fetchPromise = undefined;
         const peerBufferMap = new Map();
         const visited = new Set();
@@ -627,8 +767,8 @@ let DocumentIndex = class DocumentIndex extends Program {
         };
         const fetchFirst = async (n) => {
             done = true; // Assume we are donne
-            queryRequest.fetch = n;
-            await this.queryDetailed(queryRequest, {
+            queryRequestCoerced.fetch = n;
+            await this.queryCommence(queryRequestCoerced, {
                 ...options,
                 onResponse: async (response, from) => {
                     if (!from) {
@@ -649,18 +789,33 @@ let DocumentIndex = class DocumentIndex extends Program {
                         }
                         const buffer = [];
                         for (const result of results.results) {
-                            const indexKey = indexerTypes.toId(this.indexByResolver(result.value)).primitive;
-                            if (visited.has(indexKey)) {
-                                continue;
+                            if (result instanceof types.ResultValue) {
+                                const indexKey = indexerTypes.toId(this.indexByResolver(result.value)).primitive;
+                                if (visited.has(indexKey)) {
+                                    continue;
+                                }
+                                visited.add(indexKey);
+                                buffer.push({
+                                    value: result.value,
+                                    context: result.context,
+                                    from,
+                                    indexed: result.indexed ||
+                                        (await this.transformer(result.value, result.context)),
+                                });
+                            }
+                            else {
+                                const indexKey = indexerTypes.toId(this.indexByResolver(result.value)).primitive;
+                                if (visited.has(indexKey)) {
+                                    continue;
+                                }
+                                visited.add(indexKey);
+                                buffer.push({
+                                    value: result.value,
+                                    context: result.context,
+                                    from,
+                                    indexed: result.indexed || result.value,
+                                });
                             }
-                            visited.add(indexKey);
-                            buffer.push({
-                                value: result.value,
-                                context: result.context,
-                                from,
-                                indexed: result.indexed ||
-                                    (await this.transformer(result.value, result.context)),
-                            });
                         }
                         peerBufferMap.set(from.hashcode(), {
                             buffer,
@@ -673,7 +828,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                 },
             });
             if (done) {
-                this.clearResultsQueue(queryRequest);
+                this.clearResultsQueue(queryRequestCoerced);
             }
             return done;
         };
@@ -703,7 +858,7 @@ let DocumentIndex = class DocumentIndex extends Program {
                     // TODO buffer more than deleted?
                     // TODO batch to multiple 'to's
                     const collectRequest = new types.CollectNextRequest({
-                        id: queryRequest.id,
+                        id: queryRequestCoerced.id,
                         amount: n - buffer.buffer.length,
                     });
                     // Fetch locally?
@@ -753,11 +908,12 @@ let DocumentIndex = class DocumentIndex extends Program {
                             priority: 1,
                             mode: new SilentDelivery({ to: [peer], redundancy: 1 }),
                         })
-                            .then((response) => introduceEntries(queryRequest, response, this.documentType, this._sync, options)
-                            .then((responses) => {
-                            responses.map((response) => {
+                            .then((response) => introduceEntries(queryRequestCoerced, response, this.documentType, this.indexedType, this._sync, options)
+                            .then(async (responses) => {
+                            return Promise.all(responses.map(async (response, i) => {
                                 resultsLeft += Number(response.response.kept);
-                                if (!response.from) {
+                                const from = responses[i].from;
+                                if (!from) {
                                     logger.error("Missing from for sorted query");
                                     return;
                                 }
@@ -773,19 +929,20 @@ let DocumentIndex = class DocumentIndex extends Program {
                                     }
                                     peerBuffer.kept = Number(response.response.kept);
                                     for (const result of response.response.results) {
-                                        if (visited.has(indexerTypes.toId(this.indexByResolver(result.value)).primitive)) {
+                                        const idPrimitive = indexerTypes.toId(this.indexByResolver(result.value)).primitive;
+                                        if (visited.has(idPrimitive)) {
                                             continue;
                                         }
-                                        visited.add(indexerTypes.toId(this.indexByResolver(result.value)).primitive);
+                                        visited.add(idPrimitive);
                                         peerBuffer.buffer.push({
                                             value: result.value,
                                             context: result.context,
-                                            from: response.from,
-                                            indexed: this.transformer(result.value, result.context),
+                                            from: from,
+                                            indexed: await this.transformer(result.value, result.context),
                                         });
                                     }
                                 }
-                            });
+                            }));
                         })
                             .catch((e) => {
                             logger.error("Failed to collect sorted results from: " +
@@ -815,7 +972,7 @@ let DocumentIndex = class DocumentIndex extends Program {
             const fetchedAll = await fetchAtLeast(n);
             // get n next top entries, shift and pull more results
             const peerBuffersArr = peerBuffers();
-            const results = peerBuffersArr.sort((a, b) => indexerTypes.extractSortCompare(a.indexed, b.indexed, queryRequest.sort));
+            const results = peerBuffersArr.sort((a, b) => indexerTypes.extractSortCompare(a.indexed, b.indexed, queryRequestCoerced.sort));
             const pendingMoreResults = n < results.length;
             const batch = results.splice(0, n);
             for (const result of batch) {
@@ -830,12 +987,24 @@ let DocumentIndex = class DocumentIndex extends Program {
                 }
             }
             done = fetchedAll && !pendingMoreResults;
-            return dedup(batch.map((x) => x.value), this.indexByResolver);
+            let coercedBatch;
+            if (resolve) {
+                coercedBatch = (await Promise.all(batch.map(async (x) => x.value instanceof this.documentType
+                    ? x.value
+                    : (await this.resolveDocument({
+                        head: x.context.head,
+                        indexed: x.indexed,
+                    }))?.value))).filter((x) => !!x);
+            }
+            else {
+                coercedBatch = batch.map((x) => x.value);
+            }
+            return dedup(coercedBatch, this.indexByResolver);
         };
         const close = async () => {
             controller.abort(new AbortError("Iterator closed"));
             const closeRequest = new types.CloseIteratorRequest({
-                id: queryRequest.id,
+                id: queryRequestCoerced.id,
             });
             const promises = [];
             for (const [peer, buffer] of peerBufferMap) {
@@ -872,6 +1041,19 @@ let DocumentIndex = class DocumentIndex extends Program {
             },
         };
     }
+    async waitFor(other, options) {
+        await super.waitFor(other, options);
+        const ids = Array.isArray(other) ? other : [other];
+        const expectedHashes = new Set(ids.map((x) => typeof x === "string"
+            ? x
+            : x instanceof PublicSignKey
+                ? x.hashcode()
+                : getPublicKeyFromPeerId(x).hashcode()));
+        for (const key of expectedHashes) {
+            await waitFor(async () => (await this._log.replicationIndex.count({ query: { hash: key } })) >
+                0, options);
+        }
+    }
 };
 __decorate([
     field({ type: RPC }),