@peerbit/document 7.1.2 → 7.1.3

package/src/search.ts

@@ -26,6 +26,7 @@ import { copySerialization } from "./borsh.js";
 import { MAX_BATCH_SIZE } from "./constants.js";
 import { type Operation, isPutOperation } from "./operation.js";
 import type { ExtractArgs } from "./program.js";
+import { ResumableIterators } from "./resumable-iterator.js";
 
 const logger = loggerFn({ module: "document-index" });
 
@@ -119,7 +120,7 @@ if (!(await this.canRead(message.sender))) {
 } */
 
 export type CanSearch = (
-	request: indexerTypes.SearchRequest | indexerTypes.CollectNextRequest,
+	request: types.SearchRequest | types.CollectNextRequest,
 	from: PublicSignKey,
 ) => Promise<boolean> | boolean;
 
@@ -214,10 +215,7 @@ export class DocumentIndex<
 	D extends ReplicationDomain<any, Operation>,
 > extends Program<OpenOptions<T, I, D>> {
 	@field({ type: RPC })
-	_query: RPC<
-		indexerTypes.AbstractSearchRequest,
-		types.AbstractSearchResult<T>
-	>;
+	_query: RPC<types.AbstractSearchRequest, types.AbstractSearchResult<T>>;
 
 	// Original document representation
 	documentType: AbstractType<T>;
@@ -236,6 +234,7 @@ export class DocumentIndex<
 	private indexBy: string[];
 	private indexByResolver: (obj: any) => string | Uint8Array;
 	index: indexerTypes.Index<IDocumentWithContext<I>>;
+	private _resumableIterators: ResumableIterators<IDocumentWithContext<I>>;
 
 	// Transformation, indexer
 	/* fields: IndexableFields<T, I>; */
@@ -261,10 +260,7 @@ export class DocumentIndex<
 	>;
 
 	constructor(properties?: {
-		query?: RPC<
-			indexerTypes.AbstractSearchRequest,
-			types.AbstractSearchResult<T>
-		>;
+		query?: RPC<types.AbstractSearchRequest, types.AbstractSearchResult<T>>;
 	}) {
 		super();
 		this._query = properties?.query || new RPC();
@@ -344,12 +340,14 @@ export class DocumentIndex<
 				nested: {
 					match: (obj: any): obj is types.IDocumentStore<any> =>
 						obj instanceof this.dbType,
-					query: async (obj: types.IDocumentStore<any>, query) =>
+					iterate: async (obj: types.IDocumentStore<any>, query) =>
 						obj.index.search(query),
 				},
 				/* maxBatchSize: MAX_BATCH_SIZE */
 			})) || new HashmapIndex<IDocumentWithContext<I>>();
 
+		this._resumableIterators = new ResumableIterators(this.index);
+
 		await this._query.open({
 			topic: sha256Base64Sync(
 				concat([this._log.log.id, fromString("/document")]),
@@ -362,26 +360,24 @@ export class DocumentIndex<
 
 				if (
 					properties.canSearch &&
-					(query instanceof indexerTypes.SearchRequest ||
-						query instanceof indexerTypes.CollectNextRequest) &&
+					(query instanceof types.SearchRequest ||
+						query instanceof types.CollectNextRequest) &&
 					!(await properties.canSearch(
-						query as
-							| indexerTypes.SearchRequest
-							| indexerTypes.CollectNextRequest,
+						query as types.SearchRequest | types.CollectNextRequest,
 						ctx.from,
 					))
 				) {
 					return new types.NoAccess();
 				}
 
-				if (query instanceof indexerTypes.CloseIteratorRequest) {
+				if (query instanceof types.CloseIteratorRequest) {
 					this.processCloseIteratorRequest(query, ctx.from);
 				} else {
 					const results = await this.processQuery(
 						query as
-							| indexerTypes.SearchRequest
-							| indexerTypes.SearchRequest
-							| indexerTypes.CollectNextRequest,
+							| types.SearchRequest
+							| types.SearchRequest
+							| types.CollectNextRequest,
 						ctx.from,
 						false,
 						{
@@ -397,17 +393,17 @@ export class DocumentIndex<
 				}
 			},
 			responseType: types.AbstractSearchResult,
-			queryType: indexerTypes.AbstractSearchRequest,
+			queryType: types.AbstractSearchRequest,
 		});
 	}
 
-	getPending(cursorId: string): number | undefined {
+	async getPending(cursorId: string): Promise<number | undefined> {
 		const queue = this._resultQueue.get(cursorId);
 		if (queue) {
 			return queue.queue.length + queue.keptInIndex;
 		}
 
-		return this.index.getPending(cursorId);
+		return this._resumableIterators.getPending(cursorId);
 	}
 
 	async close(from?: Program): Promise<boolean> {
@@ -472,11 +468,9 @@ export class DocumentIndex<
 		} else {
 			this._resolverCache.del(key.primitive);
 		}
-		return this.index.del(
-			new indexerTypes.DeleteRequest({
-				query: [indexerTypes.getMatcher(this.indexBy, key.key)],
-			}),
-		);
+		return this.index.del({
+			query: [indexerTypes.getMatcher(this.indexBy, key.key)],
+		});
 	}
 
 	public async getDetailed(
@@ -486,7 +480,7 @@ export class DocumentIndex<
 		let results: types.Results<T>[] | undefined;
 		if (key instanceof Uint8Array) {
 			results = await this.queryDetailed(
-				new indexerTypes.SearchRequest({
+				new types.SearchRequest({
 					query: [
 						new indexerTypes.ByteMatchQuery({ key: this.indexBy, value: key }),
 					],
@@ -501,7 +495,7 @@ export class DocumentIndex<
 				typeof indexableKey === "bigint"
 			) {
 				results = await this.queryDetailed(
-					new indexerTypes.SearchRequest({
+					new types.SearchRequest({
 						query: [
 							new indexerTypes.IntegerCompare({
 								key: this.indexBy,
@@ -514,7 +508,7 @@ export class DocumentIndex<
 				);
 			} else if (typeof indexableKey === "string") {
 				results = await this.queryDetailed(
-					new indexerTypes.SearchRequest({
+					new types.SearchRequest({
 						query: [
 							new indexerTypes.StringMatch({
 								key: this.indexBy,
@@ -526,7 +520,7 @@ export class DocumentIndex<
 				);
 			} else if (indexableKey instanceof Uint8Array) {
 				results = await this.queryDetailed(
-					new indexerTypes.SearchRequest({
+					new types.SearchRequest({
 						query: [
 							new indexerTypes.ByteMatchQuery({
 								key: this.indexBy,
@@ -586,7 +580,7 @@ export class DocumentIndex<
 	}
 
 	async processQuery(
-		query: indexerTypes.SearchRequest | indexerTypes.CollectNextRequest,
+		query: types.SearchRequest | types.CollectNextRequest,
 		from: PublicSignKey,
 		isLocal: boolean,
 		options?: {
@@ -605,13 +599,13 @@ export class DocumentIndex<
 		let indexedResult:
 			| indexerTypes.IndexedResults<IDocumentWithContext<I>>
 			| undefined = undefined;
-		if (query instanceof indexerTypes.SearchRequest) {
-			indexedResult = await this.index.query(query);
-		} else if (query instanceof indexerTypes.CollectNextRequest) {
+		if (query instanceof types.SearchRequest) {
+			indexedResult = await this._resumableIterators.iterateAndFetch(query);
+		} else if (query instanceof types.CollectNextRequest) {
 			indexedResult =
 				prevQueued?.keptInIndex === 0
-					? { kept: 0, results: [] }
-					: await this.index.next(query);
+					? []
+					: await this._resumableIterators.next(query);
 		} else {
 			throw new Error("Unsupported");
 		}
@@ -619,14 +613,16 @@ export class DocumentIndex<
 		let resultSize = 0;
 
 		let toIterate = prevQueued
-			? [...prevQueued.queue, ...indexedResult.results]
-			: indexedResult.results;
+			? [...prevQueued.queue, ...indexedResult]
+			: indexedResult;
 
 		if (prevQueued) {
 			this._resultQueue.delete(query.idString);
 			prevQueued = undefined;
 		}
 
+		let kept = (await this._resumableIterators.getPending(query.idString)) ?? 0;
+
 		if (!isLocal) {
 			prevQueued = {
 				from,
@@ -634,7 +630,7 @@ export class DocumentIndex<
 				timeout: setTimeout(() => {
 					this._resultQueue.delete(query.idString);
 				}, 6e4),
-				keptInIndex: indexedResult.kept,
+				keptInIndex: kept,
 			};
 			this._resultQueue.set(query.idString, prevQueued);
 		}
@@ -667,7 +663,7 @@ export class DocumentIndex<
 		}
 		const results: types.Results<T> = new types.Results({
 			results: filteredResults,
-			kept: BigInt(indexedResult.kept + (prevQueued?.queue.length || 0)),
+			kept: BigInt(kept + (prevQueued?.queue.length || 0)),
 		});
 
 		if (!isLocal && results.kept === 0n) {
@@ -679,9 +675,9 @@ export class DocumentIndex<
 
 	clearResultsQueue(
 		query:
-			| indexerTypes.SearchRequest
-			| indexerTypes.CollectNextRequest
-			| indexerTypes.CloseIteratorRequest,
+			| types.SearchRequest
+			| types.CollectNextRequest
+			| types.CloseIteratorRequest,
 	) {
 		const queue = this._resultQueue.get(query.idString);
 		if (queue) {
@@ -690,7 +686,7 @@ export class DocumentIndex<
 		}
 	}
 	async processCloseIteratorRequest(
-		query: indexerTypes.CloseIteratorRequest,
+		query: types.CloseIteratorRequest,
 		publicKey: PublicSignKey,
 	): Promise<void> {
 		const queueData = this._resultQueue.get(query.idString);
@@ -699,7 +695,7 @@ export class DocumentIndex<
 			return;
 		}
 		this.clearResultsQueue(query);
-		return this.index.close(query);
+		return this._resumableIterators.close(query);
 	}
 
 	/**
@@ -709,7 +705,7 @@ export class DocumentIndex<
 	 * @returns
 	 */
 	public async queryDetailed(
-		queryRequest: indexerTypes.SearchRequest,
+		queryRequest: types.SearchRequest,
 		options?: QueryDetailedOptions<T, D>,
 	): Promise<types.Results<T>[]> {
 		const local = typeof options?.local === "boolean" ? options?.local : true;
@@ -777,7 +773,7 @@ export class DocumentIndex<
 					}
 				};
 				try {
-					if (queryRequest instanceof indexerTypes.CloseIteratorRequest) {
+					if (queryRequest instanceof types.CloseIteratorRequest) {
 						// don't wait for responses
 						await this._query.request(queryRequest, { mode: remote!.mode });
 					} else {
@@ -830,7 +826,7 @@ export class DocumentIndex<
 	 * @returns
 	 */
 	public async search(
-		queryRequest: indexerTypes.SearchRequest,
+		queryRequest: types.SearchRequest,
 		options?: SearchOptions<T, D>,
 	): Promise<T[]> {
 		// Set fetch to search size, or max value (default to max u32 (4294967295))
@@ -841,10 +837,7 @@ export class DocumentIndex<
 
 		// So that this call will not do any remote requests
 		const allResults: T[] = [];
-		while (
-			iterator.done() === false &&
-			queryRequest.fetch > allResults.length
-		) {
+		while (iterator.done() !== true && queryRequest.fetch > allResults.length) {
 			// We might need to pull .next multiple time due to data message size limitations
 			for (const result of await iterator.next(
 				queryRequest.fetch - allResults.length,
@@ -866,7 +859,7 @@ export class DocumentIndex<
 	 * @returns
 	 */
 	public iterate(
-		queryRequest: indexerTypes.SearchRequest,
+		queryRequest: types.SearchRequest,
 		options?: QueryOptions<T, D>,
 	): ResultsIterator<T> {
 		let fetchPromise: Promise<any> | undefined = undefined;
@@ -986,7 +979,7 @@ export class DocumentIndex<
 
 					// TODO buffer more than deleted?
 					// TODO batch to multiple 'to's
-					const collectRequest = new indexerTypes.CollectNextRequest({
+					const collectRequest = new types.CollectNextRequest({
 						id: queryRequest.id,
 						amount: n - buffer.buffer.length,
 					});
@@ -1179,7 +1172,7 @@ export class DocumentIndex<
 		const close = async () => {
 			controller.abort(new AbortError("Iterator closed"));
 
-			const closeRequest = new indexerTypes.CloseIteratorRequest({
+			const closeRequest = new types.CloseIteratorRequest({
 				id: queryRequest.id,
 			});
 			const promises: Promise<any>[] = [];

package/dist/benchmark/memory/insert.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=insert.d.ts.map

package/dist/benchmark/memory/insert.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"insert.d.ts","sourceRoot":"","sources":["../../../benchmark/memory/insert.ts"],"names":[],"mappings":""}

package/dist/benchmark/memory/insert.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"insert.js","sourceRoot":"","sources":["../../../benchmark/memory/insert.ts"],"names":[],"mappings":";;;;;;;;;AAAA,yEAAyE;AACzE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,OAAO,EAAsB,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,SAAS,EAAqB,MAAM,sBAAsB,CAAC;AAGpE,mEAAmE;AAEnE,oCAAoC;AAEpC,IACM,QAAQ,GADd,MACM,QAAQ;IAEb,EAAE,CAAS;IAGX,IAAI,CAAU;IAGd,MAAM,CAAU;IAGhB,KAAK,CAAa;IAElB,YAAY,IAAc;QACzB,IAAI,IAAI,EAAE,CAAC;YACV,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,CAAC;IACF,CAAC;CACD,CAAA;AAnBA;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;;oCACf;AAGX;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;;sCACpB;AAGd;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;;wCACf;AAGhB;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;8BACrB,UAAU;uCAAC;AAXb,QAAQ;IADb,OAAO,CAAC,UAAU,CAAC;qCAcD,QAAQ;GAbrB,QAAQ,CAqBb;AAED,IACM,SAAS,GADf,MACM,SAAU,SAAQ,OAAwC;IAE/D,IAAI,CAAsB;IAE1B,YAAY,UAA0C;QACrD,KAAK,EAAE,CAAC;QACR,IAAI,UAAU,EAAE,CAAC;YAChB,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;QAC7B,CAAC;IACF,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,OAAyC;QACnD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IACtD,CAAC;CACD,CAAA;AAXA;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;8BACrB,SAAS;uCAAW;AAFrB,SAAS;IADd,OAAO,CAAC,gBAAgB,CAAC;;GACpB,SAAS,CAad;AAED,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,IAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;AAEzE,IAAI,OAA0D,CAAC;AAC/D,IAAI,CAAC;IACJ,IAAI,KAAK,GAA0B,SAAS,CAAC;IAE7C,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAAgB,EAAE,EAAE;QAChD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,UAAU,EAAE;gBACjD,SAAS,EACR,OAAO,CAAC,OAAO,KAAK,WAAW;oBAC9B,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;aAC9B,CAAC,CAAC;YAEH,KAAK,GAAG,IAAI,SAAS,CAAC;gBACrB,IAAI,EAAE,IAAI,SAAS,EAAY;aAC/B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAkB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE;gBACxB,IAAI,EAAE;oBACL,SAAS,EAAE;wBACV,MAAM,EAAE,CAAC;qBACT;iBACD;aACD,CAAC,CAAC;QACJ,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC;oBACxB,EAAE,EAAE,IAAI,EAAE;oBACV,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC;iBAC/C,CAAC,CAAC;gBACH,MAAM,KAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,CAAC;QACF,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QAED,SAAS,EAAE,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,SAAS,EAAE,CAAC;IAEZ,4DAA4D;IAC5D,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACnC,IAAI,QAAQ,GAAG,CAAC,OAAgB,EAAE,EAAE;YACnC,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACjC,OAAO,EAAE,CAAC;YACX,CAAC;QACF,CAAC,CAAC;QACF,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACJ,CAAC;AAAC,OAAO,KAAU,EAAE,CAAC;IACrB,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,KAAK,EAAE,OAAO,CAAC,CAAC;AACxD,CAAC;QAAS,CAAC;IACV,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACrB,MAAM,OAAQ,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC"}