@peerbit/document 13.1.4 → 13.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/document",
-  "version": "13.1.4",
+  "version": "13.1.6",
   "description": "Document store implementation",
   "type": "module",
   "sideEffects": false,
@@ -61,18 +61,18 @@
     "uint8arrays": "^5.1.0",
     "@peerbit/cache": "3.1.0",
     "@peerbit/crypto": "3.1.2",
-    "@peerbit/document-interface": "3.2.46",
-    "@peerbit/indexer-cache": "0.2.8",
-    "@peerbit/indexer-interface": "3.0.5",
-    "@peerbit/indexer-simple": "1.2.8",
-    "@peerbit/log": "6.2.3",
-    "@peerbit/indexer-sqlite3": "3.0.8",
-    "@peerbit/program": "6.0.33",
+    "@peerbit/document-interface": "3.2.48",
+    "@peerbit/indexer-interface": "3.0.6",
+    "@peerbit/indexer-cache": "0.2.9",
+    "@peerbit/indexer-simple": "1.2.10",
+    "@peerbit/log": "6.2.5",
+    "@peerbit/indexer-sqlite3": "3.0.9",
+    "@peerbit/logger": "2.0.1",
     "@peerbit/pubsub": "5.3.0",
-    "@peerbit/shared-log": "13.2.4",
-    "@peerbit/rpc": "6.1.1",
-    "@peerbit/stream-interface": "6.0.11",
-    "@peerbit/logger": "2.0.1"
+    "@peerbit/rpc": "6.1.3",
+    "@peerbit/program": "6.0.35",
+    "@peerbit/shared-log": "13.2.6",
+    "@peerbit/stream-interface": "6.0.11"
   },
   "optionalDependencies": {
     "@peerbit/document-rust": "0.1.1"
@@ -83,11 +83,11 @@
     "@types/pidusage": "^2.0.5",
     "pidusage": "^4.0.1",
     "uuid": "^10.0.0",
-    "@peerbit/native-backbone": "0.1.3",
-    "@peerbit/test-utils": "3.1.4",
     "@peerbit/log-rust": "1.1.2",
-    "peerbit": "5.3.4",
-    "@peerbit/time": "3.0.0"
+    "@peerbit/native-backbone": "0.1.4",
+    "@peerbit/test-utils": "3.1.6",
+    "@peerbit/time": "3.0.0",
+    "peerbit": "5.3.6"
   },
   "repository": {
     "type": "git",
@@ -102,6 +102,7 @@
     "clean": "aegir clean",
     "build": "aegir build --no-bundle",
     "test": "aegir test --target node",
+    "test:document-rust-core": "npm run build && PEERBIT_SHARED_LOG_RUST_CORE=1 aegir test -t node --grep '^(?!.*(uses the validated local append path|uses the independent native prepared batch path|uses native shared-log planning for replicated target-none puts|uses commit-only local puts when coordinate persistence is deferred)).*(native conformance guard |operations basic |operations get |operations index |operations search |iterate sort |count approximate |query distribution |returnIndexed |caching |custom index |prefetch )' -- --forbid-pending",
     "lint": "aegir lint",
     "test:cov": "aegir test -t node --cov"
   }

package/src/program.ts

@@ -112,6 +112,16 @@ export class NativeDocumentModeError extends Error {
 
 type MaybePromise<T> = Promise<T> | T;
 
+/**
+ * True when an error signals that an entry's payload bytes were never
+ * materialized on the JS side (a hollow entry backed by a native block store).
+ * `DecryptedThing.getValue` throws `Error("Missing data")` in that case. Used to
+ * decide whether the auto-mode append/delete read-back should recover the
+ * operation from the storage-bytes / block-store path instead.
+ */
+const isMissingPayloadDataError = (error: unknown): boolean =>
+	error instanceof Error && error.message === "Missing data";
+
 const isPromiseLike = <T>(value: MaybePromise<T>): value is Promise<T> =>
 	!!value && typeof (value as Promise<T>).then === "function";
 
@@ -2837,17 +2847,44 @@ export class Documents<
 			return;
 		}
 		ensureInitialized?.();
-		return entry.getPayloadValue();
+		try {
+			return await entry.getPayloadValue();
+		} catch (error) {
+			// Auto (non-native document) mode with a native block store: the entry
+			// materialized in the entry index can be a hollow shell whose in-memory
+			// payload bytes were never loaded (the native store keeps the block
+			// bytes at the storage layer, not on the JS entry). `getPayloadValue`
+			// then throws "Missing data". The block itself is present, so recover
+			// the operation via the storage-bytes / block-store read path. This is
+			// a no-op for the pure-JS backend, where `getPayloadValue` succeeds.
+			if (!isMissingPayloadDataError(error)) {
+				throw error;
+			}
+			const operation = await this.getPlainEntryOperationFromStorage(entry);
+			if (operation) {
+				return operation;
+			}
+			throw error;
+		}
 	}
 
 	private async getPlainEntryOperationFromStorage(
 		entry: Entry<Operation>,
 	): Promise<Operation | undefined> {
-		let storageBytes: Uint8Array;
+		let storageBytes: Uint8Array | undefined;
 		try {
 			storageBytes =
 				Entry.getPreparedStorageBytes(entry) ?? entry.getStorageBytes();
 		} catch {
+			// The entry object is hollow (its payload bytes never materialized on
+			// the JS side), so it cannot re-serialize itself. Fall through to the
+			// block-store fallback below.
+			storageBytes = undefined;
+		}
+		// Fall back to the raw block held by the block store, keyed by the entry
+		// hash, whenever the entry could not (or did not) yield its own bytes.
+		storageBytes ??= await this.getEntryStorageBytesFromBlocks(entry);
+		if (!storageBytes) {
 			return;
 		}
 		try {
@@ -2869,6 +2906,21 @@ export class Documents<
 		}
 	}
 
+	private async getEntryStorageBytesFromBlocks(
+		entry: Entry<Operation>,
+	): Promise<Uint8Array | undefined> {
+		const hash = entry.hash;
+		if (!hash) {
+			return;
+		}
+		try {
+			const bytes = await this.log.log.blocks.get(hash);
+			return bytes ?? undefined;
+		} catch {
+			return;
+		}
+	}
+
 	private getNativeDocumentFieldExtractionPlan(
 		path: string | readonly string[],
 	): SimpleDocumentFieldExtractionPlan | undefined {

package/src/resumable-iterator.ts

@@ -11,6 +11,11 @@ import { logger as loggerFn } from "@peerbit/logger";
 const iteratorLogger = loggerFn("peerbit:document:index:iterate");
 
 export class ResumableIterators<T extends Record<string, any>> {
+	private readonly pendingMarks = new Map<
+		string,
+		Map<string, indexerTypes.IdKey>
+	>();
+
 	constructor(
 		readonly index: indexerTypes.Index<T>,
 		readonly queues = new Cache<{
@@ -22,6 +27,36 @@ export class ResumableIterators<T extends Record<string, any>> {
 		// TODO choose upper limit better
 	}
 
+	private markKey(id: indexerTypes.IdKey) {
+		return `${id.key instanceof Uint8Array ? "bytes" : typeof id.key}:${String(id.primitive)}`;
+	}
+
+	private retainMarks(id: string, ids: Iterable<indexerTypes.IdKey>) {
+		let retained = this.pendingMarks.get(id);
+		for (const mark of ids) {
+			if (!retained) {
+				retained = new Map();
+				this.pendingMarks.set(id, retained);
+			}
+			retained.set(this.markKey(mark), mark);
+		}
+	}
+
+	private async applyPendingMarks(
+		id: string,
+		iterator: indexerTypes.IndexIterator<T, undefined>,
+	) {
+		while (true) {
+			const retained = this.pendingMarks.get(id);
+			if (!retained?.size) {
+				this.pendingMarks.delete(id);
+				return;
+			}
+			this.pendingMarks.delete(id);
+			await iterator.markYielded?.(retained.values());
+		}
+	}
+
 	async iterateAndFetch(
 		request: SearchRequest | SearchRequestIndexed | IterationRequest,
 		options?: { keepAlive?: boolean },
@@ -31,29 +66,32 @@ export class ResumableIterators<T extends Record<string, any>> {
 			fetch: request.fetch,
 			keepAlive: Boolean(options?.keepAlive),
 		});
-		const iterator = this.index.iterate(request);
-		const firstResult = await iterator.next(request.fetch);
 		const keepAlive = options?.keepAlive === true;
-		if (keepAlive || iterator.done() !== true) {
-			const cachedIterator = {
-				iterator,
-				request,
+		const iterator = this.index.iterate(request);
+		this.queues.add(request.idString, { iterator, request, keepAlive });
+		try {
+			await this.applyPendingMarks(request.idString, iterator);
+			const firstResult = await iterator.next(request.fetch);
+			if (!keepAlive && iterator.done() === true) {
+				this.queues.del(request.idString);
+				this.pendingMarks.delete(request.idString);
+			}
+			iteratorLogger("iterate:queued", {
+				id: request.idString,
 				keepAlive,
-			};
-			this.queues.add(request.idString, cachedIterator);
+				done: iterator.done() === true,
+				batch: firstResult.length,
+			});
+			/* console.debug(
+				"[ResumableIterators] iterateAndFetch",
+				request.idString,
+				{ keepAlive },
+			); */
+			return firstResult;
+		} catch (error) {
+			this.clear(request.idString);
+			throw error;
 		}
-		iteratorLogger("iterate:queued", {
-			id: request.idString,
-			keepAlive,
-			done: iterator.done() === true,
-			batch: firstResult.length,
-		});
-		/* console.debug(
-			"[ResumableIterators] iterateAndFetch",
-			request.idString,
-			{ keepAlive },
-		); */
-		return firstResult;
 	}
 
 	async next(
@@ -138,6 +176,7 @@ export class ResumableIterators<T extends Record<string, any>> {
 			}
 		}
 		this.queues.del(id);
+		this.pendingMarks.delete(id);
 	}
 
 	async clearAll() {
@@ -155,6 +194,7 @@ export class ResumableIterators<T extends Record<string, any>> {
 			),
 		);
 		this.queues.clear();
+		this.pendingMarks.clear();
 	}
 
 	has(id: string) {
@@ -179,4 +219,12 @@ export class ResumableIterators<T extends Record<string, any>> {
 		});
 		return pending;
 	}
+
+	async markYielded(id: string, ids: Iterable<indexerTypes.IdKey>) {
+		this.retainMarks(id, ids);
+		const iterator = this.queues.get(id);
+		if (iterator) {
+			await this.applyPendingMarks(id, iterator.iterator);
+		}
+	}
 }

package/src/search.ts

@@ -24,7 +24,7 @@ import {
 import { CachedIndex, type QueryCacheOptions } from "@peerbit/indexer-cache";
 import * as indexerTypes from "@peerbit/indexer-interface";
 import { HashmapIndex } from "@peerbit/indexer-simple";
-import { BORSH_ENCODING, type Encoding, Entry } from "@peerbit/log";
+import { BORSH_ENCODING, type Encoding, type Entry } from "@peerbit/log";
 import { logger as loggerFn } from "@peerbit/logger";
 import { ClosedError, Program } from "@peerbit/program";
 import {
@@ -1179,13 +1179,98 @@ export class DocumentIndex<
 		}
 	}
 
+	/**
+	 * Resolve documents without bypassing the resolver/program caches. Cache and
+	 * identity-index hits are collected first; only the remaining heads cross the
+	 * log read boundary, and those reads stay aligned in one batch.
+	 */
+	private async resolveDocumentsWithBatchedHeads(
+		values: Array<{
+			id?: indexerTypes.IdPrimitive;
+			indexed: I;
+			head: string;
+		}>,
+	): Promise<{
+		resolved: Array<{ value: T } | undefined>;
+		heads: Array<Entry<Operation> | undefined>;
+	}> {
+		const resolved: Array<{ value: T } | undefined> = new Array(values.length);
+		const heads: Array<Entry<Operation> | undefined> = new Array(values.length);
+		const unresolvedPositions: number[] = [];
+
+		for (let i = 0; i < values.length; i++) {
+			const value = values[i]!;
+			const cached = await this.resolveDocument({
+				...value,
+				headEntry: null,
+			});
+			if (cached) {
+				resolved[i] = cached;
+			} else {
+				unresolvedPositions.push(i);
+			}
+		}
+
+		if (unresolvedPositions.length === 0) {
+			return { resolved, heads };
+		}
+
+		const unresolvedHeads = await this._log.log.getMany(
+			unresolvedPositions.map((position) => values[position]!.head),
+		);
+		for (let i = 0; i < unresolvedPositions.length; i++) {
+			const position = unresolvedPositions[i]!;
+			const head = unresolvedHeads[i];
+			heads[position] = head;
+			resolved[position] = await this.resolveDocument({
+				...values[position]!,
+				headEntry: head ?? null,
+			});
+		}
+
+		return { resolved, heads };
+	}
+
 	private async wrapPushResults(
 		matches: Array<WithContext<T> | WithContext<I>>,
 		resolve: boolean,
 	): Promise<types.Result[]> {
 		if (!matches.length) return [];
+		const headsByMatch: Array<Entry<Operation> | undefined> = [];
+		const resolvedByMatch: Array<{ value: T } | undefined> = [];
+		if (!resolve) {
+			headsByMatch.push(
+				...(await this._log.log.getMany(
+					matches.map((match) => match.__context.head),
+				)),
+			);
+		} else {
+			const indexedPositions: number[] = [];
+			for (let i = 0; i < matches.length; i++) {
+				if (!(matches[i] instanceof this.documentType)) {
+					indexedPositions.push(i);
+				}
+			}
+			const indexedBatch = indexedPositions.length
+				? await this.resolveDocumentsWithBatchedHeads(
+						indexedPositions.map((position) => {
+							const indexed = matches[position] as WithContext<I>;
+							return {
+								indexed,
+								head: indexed.__context.head,
+							};
+						}),
+					)
+				: { resolved: [], heads: [] };
+			for (let i = 0; i < indexedPositions.length; i++) {
+				const position = indexedPositions[i]!;
+				resolvedByMatch[position] = indexedBatch.resolved[i];
+				headsByMatch[position] = indexedBatch.heads[i];
+			}
+		}
 		const results: types.Result[] = [];
-		for (const match of matches) {
+		for (let i = 0; i < matches.length; i++) {
+			const match = matches[i]!;
 			if (resolve) {
 				if (match instanceof this.documentType) {
 					const doc = match as WithContext<T>;
@@ -1203,10 +1288,7 @@ export class DocumentIndex<
 				}
 
 				const indexed = match as WithContext<I>;
-				const resolved = await this.resolveDocument({
-					indexed,
-					head: indexed.__context.head,
-				});
+				const resolved = resolvedByMatch[i];
 
 				if (resolved) {
 					results.push(
@@ -1220,7 +1302,7 @@ export class DocumentIndex<
 					continue;
 				}
 
-				const head = await this._log.log.get(indexed.__context.head);
+				const head = headsByMatch[i];
 				results.push(
 					new types.ResultIndexedValue({
 						context: indexed.__context,
@@ -1231,7 +1313,7 @@ export class DocumentIndex<
 				);
 			} else {
 				const indexed = match as WithContext<I>;
-				const head = await this._log.log.get(indexed.__context.head);
+				const head = headsByMatch[i];
 				results.push(
 					new types.ResultIndexedValue({
 						context: indexed.__context,
@@ -1253,17 +1335,28 @@ export class DocumentIndex<
 			return [];
 		}
 		const drained = queueEntries.splice(0);
+		const resolvedBatch = resolve
+			? await this.resolveDocumentsWithBatchedHeads(
+					drained.map((entry) => ({
+						indexed: entry.value,
+						head: entry.value.__context.head,
+					})),
+				)
+			: undefined;
+		const heads = resolvedBatch
+			? resolvedBatch.heads
+			: await this._log.log.getMany(
+					drained.map((entry) => entry.value.__context.head),
+				);
 		const results: types.Result[] = [];
-		for (const entry of drained) {
+		for (let i = 0; i < drained.length; i++) {
+			const entry = drained[i]!;
 			const indexedUnwrapped = Object.assign(
 				Object.create(this.indexedType.prototype),
 				entry.value,
 			);
 			if (resolve) {
-				const value = await this.resolveDocument({
-					indexed: entry.value,
-					head: entry.value.__context.head,
-				});
+				const value = resolvedBatch!.resolved[i];
 				if (value) {
 					results.push(
 						new types.ResultValue({
@@ -1276,7 +1369,7 @@ export class DocumentIndex<
 					continue;
 				}
 
-				const head = await this._log.log.get(entry.value.__context.head);
+				const head = heads[i];
 				results.push(
 					new types.ResultIndexedValue({
 						context: entry.value.__context,
@@ -1286,7 +1379,7 @@ export class DocumentIndex<
 					}),
 				);
 			} else {
-				const head = await this._log.log.get(entry.value.__context.head);
+				const head = heads[i];
 				results.push(
 					new types.ResultIndexedValue({
 						context: entry.value.__context,
@@ -1336,6 +1429,7 @@ export class DocumentIndex<
 				let pendingAdded = added;
 				do {
 					const batches: types.Result[] = [];
+					let claimedIds: indexerTypes.IdKey[] = [];
 					const queued = await this.drainQueuedResults(
 						queue.queue,
 						resolveFlag,
@@ -1357,6 +1451,21 @@ export class DocumentIndex<
 						const wrapped = await this.wrapPushResults(matches, resolveFlag);
 						if (wrapped.length) {
 							batches.push(...wrapped);
+							claimedIds = wrapped
+								.map((result) =>
+									result instanceof types.ResultValue && result.indexed
+										? indexerTypes.toId(
+												this.indexByResolver(result.indexed),
+											)
+										: result instanceof types.ResultIndexedValue
+											? indexerTypes.toId(
+													this.indexByResolver(result.value),
+												)
+											: undefined,
+								)
+								.filter(
+									(id): id is indexerTypes.IdKey => id !== undefined,
+								);
 						}
 					}
 					if (batches.length) {
@@ -1374,6 +1483,12 @@ export class DocumentIndex<
 								redundancy: 1,
 							}),
 						});
+						if (claimedIds.length) {
+							await this._resumableIterators.markYielded(
+								_iteratorId,
+								claimedIds,
+							);
+						}
 					}
 					pendingAdded = queue.pendingAdded ?? [];
 					queue.pendingAdded = undefined;
@@ -3452,6 +3567,7 @@ export class DocumentIndex<
 		id?: indexerTypes.IdPrimitive;
 		indexed: I;
 		head: string;
+		headEntry?: Entry<Operation> | null;
 	}): Promise<{ value: T } | undefined> {
 		const id =
 			value.id ??
@@ -3473,7 +3589,10 @@ export class DocumentIndex<
 			return { value: obj as T };
 		}
 
-		const head = await this._log.log.get(value.head);
+		const head =
+			value.headEntry === undefined
+				? await this._log.log.get(value.head)
+				: (value.headEntry ?? undefined);
 		if (!head) {
 			return undefined; // we could end up here if we recently pruned the document and other peers never persisted the entry
 			// TODO update changes in index before removing entries from log entry storage
@@ -3611,7 +3730,10 @@ export class DocumentIndex<
 			this._resultQueue.set(query.idString, prevQueued);
 		}
 
-		const filteredResults: types.Result[] = [];
+		const candidates: Array<{
+			result: indexerTypes.IndexedResult<WithContext<I>>;
+			indexed: I;
+		}> = [];
 		const resolveDocumentsFlag = resolvesDocuments(fromQuery);
 		const replicateIndexFlag = replicatesIndex(fromQuery);
 		for (const result of toIterate) {
@@ -3633,11 +3755,29 @@ export class DocumentIndex<
 			) {
 				continue;
 			}
+			candidates.push({ result, indexed: indexedUnwrapped });
+		}
+
+		const resolvedBatch = resolveDocumentsFlag
+			? await this.resolveDocumentsWithBatchedHeads(
+					candidates.map(({ result }) => ({
+						indexed: result.value,
+						head: result.value.__context.head,
+					})),
+				)
+			: undefined;
+		const heads = resolvedBatch
+			? resolvedBatch.heads
+			: candidates.length > 0
+				? await this._log.log.getMany(
+						candidates.map(({ result }) => result.value.__context.head),
+					)
+				: [];
+		const filteredResults: types.Result[] = [];
+		for (let i = 0; i < candidates.length; i++) {
+			const { result, indexed: indexedUnwrapped } = candidates[i]!;
 			if (resolveDocumentsFlag) {
-				const value = await this.resolveDocument({
-					indexed: result.value,
-					head: result.value.__context.head,
-				});
+				const value = resolvedBatch!.resolved[i];
 
 				if (!value) {
 					continue;
@@ -3653,7 +3793,7 @@ export class DocumentIndex<
 				);
 			} else {
 				const context = result.value.__context;
-				const head = await this._log.log.get(context.head);
+				const head = heads[i];
 				if (replicateIndexFlag) {
 					if (!head) {
 						continue;
@@ -6122,9 +6262,10 @@ export class DocumentIndex<
 									continue;
 								}
 							}
-							const id = indexerTypes.toId(
+							const indexId = indexerTypes.toId(
 								this.indexByResolver(indexedCandidate),
-							).primitive;
+							);
+							const id = indexId.primitive;
 							const existingIndexed = indexedPlaceholders?.get(id);
 							if (existingIndexed) {
 								if (resolve) {
@@ -6165,6 +6306,10 @@ export class DocumentIndex<
 							if (!resolve) {
 								ensureIndexedPlaceholders().set(id, placeholder);
 							}
+							await this._resumableIterators.markYielded(
+								queryRequestCoerced.idString,
+								[indexId],
+							);
 							hasRelevantChange = true;
 						}
 					}