@peerbit/document 13.1.5 → 13.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/document",
-  "version": "13.1.5",
+  "version": "13.1.6",
   "description": "Document store implementation",
   "type": "module",
   "sideEffects": false,
@@ -59,19 +59,19 @@
     "@multiformats/multiaddr": "^13.0.1",
     "p-defer": "^4.0.0",
     "uint8arrays": "^5.1.0",
-    "@peerbit/indexer-cache": "0.2.8",
-    "@peerbit/indexer-interface": "3.0.5",
     "@peerbit/cache": "3.1.0",
-    "@peerbit/document-interface": "3.2.47",
-    "@peerbit/indexer-simple": "1.2.9",
     "@peerbit/crypto": "3.1.2",
-    "@peerbit/log": "6.2.4",
+    "@peerbit/document-interface": "3.2.48",
+    "@peerbit/indexer-interface": "3.0.6",
+    "@peerbit/indexer-cache": "0.2.9",
+    "@peerbit/indexer-simple": "1.2.10",
+    "@peerbit/log": "6.2.5",
+    "@peerbit/indexer-sqlite3": "3.0.9",
     "@peerbit/logger": "2.0.1",
-    "@peerbit/indexer-sqlite3": "3.0.8",
     "@peerbit/pubsub": "5.3.0",
-    "@peerbit/program": "6.0.34",
-    "@peerbit/rpc": "6.1.2",
-    "@peerbit/shared-log": "13.2.5",
+    "@peerbit/rpc": "6.1.3",
+    "@peerbit/program": "6.0.35",
+    "@peerbit/shared-log": "13.2.6",
     "@peerbit/stream-interface": "6.0.11"
   },
   "optionalDependencies": {
@@ -85,9 +85,9 @@
     "uuid": "^10.0.0",
     "@peerbit/log-rust": "1.1.2",
     "@peerbit/native-backbone": "0.1.4",
+    "@peerbit/test-utils": "3.1.6",
     "@peerbit/time": "3.0.0",
-    "@peerbit/test-utils": "3.1.5",
-    "peerbit": "5.3.5"
+    "peerbit": "5.3.6"
   },
   "repository": {
     "type": "git",
@@ -102,7 +102,7 @@
     "clean": "aegir clean",
     "build": "aegir build --no-bundle",
     "test": "aegir test --target node",
-    "test:document-rust-core": "npm run build && PEERBIT_SHARED_LOG_RUST_CORE=1 aegir test -t node --grep '^(?!.*(uses the validated local append path|uses the independent native prepared batch path|uses native shared-log planning for replicated target-none puts|uses commit-only local puts when coordinate persistence is deferred|can delete without being replicator)).*(native conformance guard |operations basic |operations get |operations index |operations search |iterate sort |count approximate |query distribution |returnIndexed |caching |custom index |prefetch )'",
+    "test:document-rust-core": "npm run build && PEERBIT_SHARED_LOG_RUST_CORE=1 aegir test -t node --grep '^(?!.*(uses the validated local append path|uses the independent native prepared batch path|uses native shared-log planning for replicated target-none puts|uses commit-only local puts when coordinate persistence is deferred)).*(native conformance guard |operations basic |operations get |operations index |operations search |iterate sort |count approximate |query distribution |returnIndexed |caching |custom index |prefetch )' -- --forbid-pending",
     "lint": "aegir lint",
     "test:cov": "aegir test -t node --cov"
   }

package/src/resumable-iterator.ts

@@ -11,6 +11,11 @@ import { logger as loggerFn } from "@peerbit/logger";
 const iteratorLogger = loggerFn("peerbit:document:index:iterate");
 
 export class ResumableIterators<T extends Record<string, any>> {
+	private readonly pendingMarks = new Map<
+		string,
+		Map<string, indexerTypes.IdKey>
+	>();
+
 	constructor(
 		readonly index: indexerTypes.Index<T>,
 		readonly queues = new Cache<{
@@ -22,6 +27,36 @@ export class ResumableIterators<T extends Record<string, any>> {
 		// TODO choose upper limit better
 	}
 
+	private markKey(id: indexerTypes.IdKey) {
+		return `${id.key instanceof Uint8Array ? "bytes" : typeof id.key}:${String(id.primitive)}`;
+	}
+
+	private retainMarks(id: string, ids: Iterable<indexerTypes.IdKey>) {
+		let retained = this.pendingMarks.get(id);
+		for (const mark of ids) {
+			if (!retained) {
+				retained = new Map();
+				this.pendingMarks.set(id, retained);
+			}
+			retained.set(this.markKey(mark), mark);
+		}
+	}
+
+	private async applyPendingMarks(
+		id: string,
+		iterator: indexerTypes.IndexIterator<T, undefined>,
+	) {
+		while (true) {
+			const retained = this.pendingMarks.get(id);
+			if (!retained?.size) {
+				this.pendingMarks.delete(id);
+				return;
+			}
+			this.pendingMarks.delete(id);
+			await iterator.markYielded?.(retained.values());
+		}
+	}
+
 	async iterateAndFetch(
 		request: SearchRequest | SearchRequestIndexed | IterationRequest,
 		options?: { keepAlive?: boolean },
@@ -31,29 +66,32 @@ export class ResumableIterators<T extends Record<string, any>> {
 			fetch: request.fetch,
 			keepAlive: Boolean(options?.keepAlive),
 		});
-		const iterator = this.index.iterate(request);
-		const firstResult = await iterator.next(request.fetch);
 		const keepAlive = options?.keepAlive === true;
-		if (keepAlive || iterator.done() !== true) {
-			const cachedIterator = {
-				iterator,
-				request,
+		const iterator = this.index.iterate(request);
+		this.queues.add(request.idString, { iterator, request, keepAlive });
+		try {
+			await this.applyPendingMarks(request.idString, iterator);
+			const firstResult = await iterator.next(request.fetch);
+			if (!keepAlive && iterator.done() === true) {
+				this.queues.del(request.idString);
+				this.pendingMarks.delete(request.idString);
+			}
+			iteratorLogger("iterate:queued", {
+				id: request.idString,
 				keepAlive,
-			};
-			this.queues.add(request.idString, cachedIterator);
+				done: iterator.done() === true,
+				batch: firstResult.length,
+			});
+			/* console.debug(
+				"[ResumableIterators] iterateAndFetch",
+				request.idString,
+				{ keepAlive },
+			); */
+			return firstResult;
+		} catch (error) {
+			this.clear(request.idString);
+			throw error;
 		}
-		iteratorLogger("iterate:queued", {
-			id: request.idString,
-			keepAlive,
-			done: iterator.done() === true,
-			batch: firstResult.length,
-		});
-		/* console.debug(
-			"[ResumableIterators] iterateAndFetch",
-			request.idString,
-			{ keepAlive },
-		); */
-		return firstResult;
 	}
 
 	async next(
@@ -138,6 +176,7 @@ export class ResumableIterators<T extends Record<string, any>> {
 			}
 		}
 		this.queues.del(id);
+		this.pendingMarks.delete(id);
 	}
 
 	async clearAll() {
@@ -155,6 +194,7 @@ export class ResumableIterators<T extends Record<string, any>> {
 			),
 		);
 		this.queues.clear();
+		this.pendingMarks.clear();
 	}
 
 	has(id: string) {
@@ -179,4 +219,12 @@ export class ResumableIterators<T extends Record<string, any>> {
 		});
 		return pending;
 	}
+
+	async markYielded(id: string, ids: Iterable<indexerTypes.IdKey>) {
+		this.retainMarks(id, ids);
+		const iterator = this.queues.get(id);
+		if (iterator) {
+			await this.applyPendingMarks(id, iterator.iterator);
+		}
+	}
 }

package/src/search.ts

@@ -24,7 +24,7 @@ import {
 import { CachedIndex, type QueryCacheOptions } from "@peerbit/indexer-cache";
 import * as indexerTypes from "@peerbit/indexer-interface";
 import { HashmapIndex } from "@peerbit/indexer-simple";
-import { BORSH_ENCODING, type Encoding, Entry } from "@peerbit/log";
+import { BORSH_ENCODING, type Encoding, type Entry } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import { ClosedError, Program } from "@peerbit/program";
 import {
@@ -1179,26 +1179,56 @@ export class DocumentIndex<
 		}
 	}
 
-	/** Head entry safe to borsh-serialize into ResultIndexedValue.entries over the
-	 * document RPC. Under the native block store, this._log.log.get can return a
-	 * hollow entry whose payload bytes were never materialized on the JS side;
-	 * serializing it throws and aborts the whole RPC response. The complete entry is
-	 * in the block store keyed by hash, so recover it there. No-op for pure JS. */
-	private async getSerializableHead(
-		hash: string,
-	): Promise<Entry<any> | undefined> {
-		const head = await this._log.log.get(hash);
-		if (!head?.hash) return head ?? undefined;
-		try {
-			head.getStorageBytes(); // = serialize(head); throws on a hollow native entry
-			return head;
-		} catch {
-			try {
-				return await Entry.fromMultihash(this._log.log.blocks, head.hash);
-			} catch {
-				return head; // block absent (e.g. pruned) — preserve today's behavior
+	/**
+	 * Resolve documents without bypassing the resolver/program caches. Cache and
+	 * identity-index hits are collected first; only the remaining heads cross the
+	 * log read boundary, and those reads stay aligned in one batch.
+	 */
+	private async resolveDocumentsWithBatchedHeads(
+		values: Array<{
+			id?: indexerTypes.IdPrimitive;
+			indexed: I;
+			head: string;
+		}>,
+	): Promise<{
+		resolved: Array<{ value: T } | undefined>;
+		heads: Array<Entry<Operation> | undefined>;
+	}> {
+		const resolved: Array<{ value: T } | undefined> = new Array(values.length);
+		const heads: Array<Entry<Operation> | undefined> = new Array(values.length);
+		const unresolvedPositions: number[] = [];
+
+		for (let i = 0; i < values.length; i++) {
+			const value = values[i]!;
+			const cached = await this.resolveDocument({
+				...value,
+				headEntry: null,
+			});
+			if (cached) {
+				resolved[i] = cached;
+			} else {
+				unresolvedPositions.push(i);
 			}
 		}
+
+		if (unresolvedPositions.length === 0) {
+			return { resolved, heads };
+		}
+
+		const unresolvedHeads = await this._log.log.getMany(
+			unresolvedPositions.map((position) => values[position]!.head),
+		);
+		for (let i = 0; i < unresolvedPositions.length; i++) {
+			const position = unresolvedPositions[i]!;
+			const head = unresolvedHeads[i];
+			heads[position] = head;
+			resolved[position] = await this.resolveDocument({
+				...values[position]!,
+				headEntry: head ?? null,
+			});
+		}
+
+		return { resolved, heads };
 	}
 
 	private async wrapPushResults(
@@ -1206,8 +1236,41 @@ export class DocumentIndex<
 		resolve: boolean,
 	): Promise<types.Result[]> {
 		if (!matches.length) return [];
+		const headsByMatch: Array<Entry<Operation> | undefined> = [];
+		const resolvedByMatch: Array<{ value: T } | undefined> = [];
+		if (!resolve) {
+			headsByMatch.push(
+				...(await this._log.log.getMany(
+					matches.map((match) => match.__context.head),
+				)),
+			);
+		} else {
+			const indexedPositions: number[] = [];
+			for (let i = 0; i < matches.length; i++) {
+				if (!(matches[i] instanceof this.documentType)) {
+					indexedPositions.push(i);
+				}
+			}
+			const indexedBatch = indexedPositions.length
+				? await this.resolveDocumentsWithBatchedHeads(
+						indexedPositions.map((position) => {
+							const indexed = matches[position] as WithContext<I>;
+							return {
+								indexed,
+								head: indexed.__context.head,
+							};
+						}),
+					)
+				: { resolved: [], heads: [] };
+			for (let i = 0; i < indexedPositions.length; i++) {
+				const position = indexedPositions[i]!;
+				resolvedByMatch[position] = indexedBatch.resolved[i];
+				headsByMatch[position] = indexedBatch.heads[i];
+			}
+		}
 		const results: types.Result[] = [];
-		for (const match of matches) {
+		for (let i = 0; i < matches.length; i++) {
+			const match = matches[i]!;
 			if (resolve) {
 				if (match instanceof this.documentType) {
 					const doc = match as WithContext<T>;
@@ -1225,10 +1288,7 @@ export class DocumentIndex<
 				}
 
 				const indexed = match as WithContext<I>;
-				const resolved = await this.resolveDocument({
-					indexed,
-					head: indexed.__context.head,
-				});
+				const resolved = resolvedByMatch[i];
 
 				if (resolved) {
 					results.push(
@@ -1242,7 +1302,7 @@ export class DocumentIndex<
 					continue;
 				}
 
-				const head = await this.getSerializableHead(indexed.__context.head);
+				const head = headsByMatch[i];
 				results.push(
 					new types.ResultIndexedValue({
 						context: indexed.__context,
@@ -1253,7 +1313,7 @@ export class DocumentIndex<
 				);
 			} else {
 				const indexed = match as WithContext<I>;
-				const head = await this.getSerializableHead(indexed.__context.head);
+				const head = headsByMatch[i];
 				results.push(
 					new types.ResultIndexedValue({
 						context: indexed.__context,
@@ -1275,17 +1335,28 @@ export class DocumentIndex<
 			return [];
 		}
 		const drained = queueEntries.splice(0);
+		const resolvedBatch = resolve
+			? await this.resolveDocumentsWithBatchedHeads(
+					drained.map((entry) => ({
+						indexed: entry.value,
+						head: entry.value.__context.head,
+					})),
+				)
+			: undefined;
+		const heads = resolvedBatch
+			? resolvedBatch.heads
+			: await this._log.log.getMany(
+					drained.map((entry) => entry.value.__context.head),
+				);
 		const results: types.Result[] = [];
-		for (const entry of drained) {
+		for (let i = 0; i < drained.length; i++) {
+			const entry = drained[i]!;
 			const indexedUnwrapped = Object.assign(
 				Object.create(this.indexedType.prototype),
 				entry.value,
 			);
 			if (resolve) {
-				const value = await this.resolveDocument({
-					indexed: entry.value,
-					head: entry.value.__context.head,
-				});
+				const value = resolvedBatch!.resolved[i];
 				if (value) {
 					results.push(
 						new types.ResultValue({
@@ -1298,7 +1369,7 @@ export class DocumentIndex<
 					continue;
 				}
 
-				const head = await this.getSerializableHead(entry.value.__context.head);
+				const head = heads[i];
 				results.push(
 					new types.ResultIndexedValue({
 						context: entry.value.__context,
@@ -1308,7 +1379,7 @@ export class DocumentIndex<
 					}),
 				);
 			} else {
-				const head = await this.getSerializableHead(entry.value.__context.head);
+				const head = heads[i];
 				results.push(
 					new types.ResultIndexedValue({
 						context: entry.value.__context,
@@ -1358,6 +1429,7 @@ export class DocumentIndex<
 				let pendingAdded = added;
 				do {
 					const batches: types.Result[] = [];
+					let claimedIds: indexerTypes.IdKey[] = [];
 					const queued = await this.drainQueuedResults(
 						queue.queue,
 						resolveFlag,
@@ -1379,6 +1451,21 @@ export class DocumentIndex<
 						const wrapped = await this.wrapPushResults(matches, resolveFlag);
 						if (wrapped.length) {
 							batches.push(...wrapped);
+							claimedIds = wrapped
+								.map((result) =>
+									result instanceof types.ResultValue && result.indexed
+										? indexerTypes.toId(
+												this.indexByResolver(result.indexed),
+											)
+										: result instanceof types.ResultIndexedValue
+											? indexerTypes.toId(
+													this.indexByResolver(result.value),
+												)
+											: undefined,
+								)
+								.filter(
+									(id): id is indexerTypes.IdKey => id !== undefined,
+								);
 						}
 					}
 					if (batches.length) {
@@ -1396,6 +1483,12 @@ export class DocumentIndex<
 								redundancy: 1,
 							}),
 						});
+						if (claimedIds.length) {
+							await this._resumableIterators.markYielded(
+								_iteratorId,
+								claimedIds,
+							);
+						}
 					}
 					pendingAdded = queue.pendingAdded ?? [];
 					queue.pendingAdded = undefined;
@@ -3474,6 +3567,7 @@ export class DocumentIndex<
 		id?: indexerTypes.IdPrimitive;
 		indexed: I;
 		head: string;
+		headEntry?: Entry<Operation> | null;
 	}): Promise<{ value: T } | undefined> {
 		const id =
 			value.id ??
@@ -3495,7 +3589,10 @@ export class DocumentIndex<
 			return { value: obj as T };
 		}
 
-		const head = await this._log.log.get(value.head);
+		const head =
+			value.headEntry === undefined
+				? await this._log.log.get(value.head)
+				: (value.headEntry ?? undefined);
 		if (!head) {
 			return undefined; // we could end up here if we recently pruned the document and other peers never persisted the entry
 			// TODO update changes in index before removing entries from log entry storage
@@ -3633,7 +3730,10 @@ export class DocumentIndex<
 			this._resultQueue.set(query.idString, prevQueued);
 		}
 
-		const filteredResults: types.Result[] = [];
+		const candidates: Array<{
+			result: indexerTypes.IndexedResult<WithContext<I>>;
+			indexed: I;
+		}> = [];
 		const resolveDocumentsFlag = resolvesDocuments(fromQuery);
 		const replicateIndexFlag = replicatesIndex(fromQuery);
 		for (const result of toIterate) {
@@ -3655,11 +3755,29 @@ export class DocumentIndex<
 			) {
 				continue;
 			}
+			candidates.push({ result, indexed: indexedUnwrapped });
+		}
+
+		const resolvedBatch = resolveDocumentsFlag
+			? await this.resolveDocumentsWithBatchedHeads(
+					candidates.map(({ result }) => ({
+						indexed: result.value,
+						head: result.value.__context.head,
+					})),
+				)
+			: undefined;
+		const heads = resolvedBatch
+			? resolvedBatch.heads
+			: candidates.length > 0
+				? await this._log.log.getMany(
+						candidates.map(({ result }) => result.value.__context.head),
+					)
+				: [];
+		const filteredResults: types.Result[] = [];
+		for (let i = 0; i < candidates.length; i++) {
+			const { result, indexed: indexedUnwrapped } = candidates[i]!;
 			if (resolveDocumentsFlag) {
-				const value = await this.resolveDocument({
-					indexed: result.value,
-					head: result.value.__context.head,
-				});
+				const value = resolvedBatch!.resolved[i];
 
 				if (!value) {
 					continue;
@@ -3675,7 +3793,7 @@ export class DocumentIndex<
 				);
 			} else {
 				const context = result.value.__context;
-				const head = await this.getSerializableHead(context.head);
+				const head = heads[i];
 				if (replicateIndexFlag) {
 					if (!head) {
 						continue;
@@ -6144,9 +6262,10 @@ export class DocumentIndex<
 									continue;
 								}
 							}
-							const id = indexerTypes.toId(
+							const indexId = indexerTypes.toId(
 								this.indexByResolver(indexedCandidate),
-							).primitive;
+							);
+							const id = indexId.primitive;
 							const existingIndexed = indexedPlaceholders?.get(id);
 							if (existingIndexed) {
 								if (resolve) {
@@ -6187,6 +6306,10 @@ export class DocumentIndex<
 							if (!resolve) {
 								ensureIndexedPlaceholders().set(id, placeholder);
 							}
+							await this._resumableIterators.markYielded(
+								queryRequestCoerced.idString,
+								[indexId],
+							);
 							hasRelevantChange = true;
 						}
 					}