@peerbit/document 15.0.4 → 15.0.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/document",
-  "version": "15.0.4",
+  "version": "15.0.6",
   "description": "Document store implementation",
   "type": "module",
   "sideEffects": false,
@@ -55,40 +55,40 @@
   "dependencies": {
     "@chainsafe/libp2p-yamux": "^8.0.1",
     "@dao-xyz/borsh": "^6.0.0",
-    "@libp2p/interface": "^3.1.1",
-    "@libp2p/tcp": "^11.0.14",
-    "@multiformats/multiaddr": "^13.0.1",
+    "@libp2p/interface": "^3.2.5",
+    "@libp2p/tcp": "^11.0.26",
+    "@multiformats/multiaddr": "^13.0.3",
     "p-defer": "^4.0.0",
     "uint8arrays": "^5.1.0",
     "@peerbit/cache": "3.1.1",
-    "@peerbit/crypto": "3.1.5",
-    "@peerbit/document-interface": "3.2.60",
-    "@peerbit/indexer-cache": "0.2.12",
-    "@peerbit/indexer-interface": "3.0.9",
-    "@peerbit/indexer-simple": "1.2.13",
-    "@peerbit/indexer-sqlite3": "3.0.14",
-    "@peerbit/log": "6.2.17",
-    "@peerbit/logger": "2.0.1",
-    "@peerbit/program": "6.0.49",
-    "@peerbit/rpc": "6.1.17",
-    "@peerbit/shared-log": "16.0.4",
-    "@peerbit/time": "3.0.1",
-    "@peerbit/pubsub": "5.4.1",
-    "@peerbit/stream-interface": "6.0.15"
+    "@peerbit/crypto": "3.1.6",
+    "@peerbit/document-interface": "3.2.62",
+    "@peerbit/indexer-interface": "3.0.10",
+    "@peerbit/indexer-cache": "0.2.13",
+    "@peerbit/indexer-simple": "1.2.14",
+    "@peerbit/indexer-sqlite3": "3.0.15",
+    "@peerbit/logger": "2.0.2",
+    "@peerbit/program": "6.0.51",
+    "@peerbit/pubsub": "5.4.2",
+    "@peerbit/log": "6.2.19",
+    "@peerbit/rpc": "6.1.19",
+    "@peerbit/shared-log": "16.0.6",
+    "@peerbit/stream-interface": "6.0.16",
+    "@peerbit/time": "3.0.1"
   },
   "optionalDependencies": {
     "@peerbit/document-rust": "0.1.1"
   },
   "devDependencies": {
     "@chainsafe/libp2p-noise": "^17.0.0",
-    "@libp2p/websockets": "^10.1.7",
+    "@libp2p/websockets": "^10.1.19",
     "@types/pidusage": "^2.0.5",
     "pidusage": "^4.0.1",
     "uuid": "^11.1.1",
     "@peerbit/log-rust": "1.1.3",
-    "@peerbit/native-backbone": "0.2.5",
-    "@peerbit/test-utils": "3.1.23",
-    "peerbit": "5.3.23"
+    "@peerbit/native-backbone": "0.2.6",
+    "@peerbit/test-utils": "3.1.25",
+    "peerbit": "5.3.25"
   },
   "repository": {
     "type": "git",

package/src/program.ts

@@ -1991,6 +1991,75 @@ export class Documents<
 		>;
 	}
 
+	/**
+	 * The change-delivery variant of `getLocalIndexedContext`, and the ONLY caller
+	 * allowed to treat a closed index as "no prior version".
+	 *
+	 * `getLocalIndexedContext` reaches the backing indexer directly, bypassing
+	 * `DocumentIndex`'s guarded wrappers. `terminalChildOrder` now tears the index
+	 * down after the log, but a change can still be delivered against a closed
+	 * index on paths that do not go through that ordering: a `DocumentIndex` closed
+	 * on its own rather than through its parent, and a log teardown that fails,
+	 * which opens the rank gate without having drained. For a change that the log
+	 * has ALREADY committed, "there is no prior version to compare against" is the
+	 * right reading, and the write leg - which already returns without persisting
+	 * once the index is closed, see `DocumentIndex.putWithContext` - drops it.
+	 *
+	 * WHY THIS IS NOT ON THE SHARED HELPER. It was, and that was wrong. The helper
+	 * has five callers and only this one is a post-commit change:
+	 *
+	 *   :2689 resolveCanPerformDeleteValue - feeds the user's canPerform policy
+	 *   :2794 _canAppend PUT   - immutable-overwrite protection
+	 *   :2863 _canAppend DELETE - delete-history validation
+	 *   :3124 putCompatDocumentBackend - the user-initiated put() path
+	 *
+	 * Swallowing there is actively harmful rather than merely broad. On :3124 a
+	 * put() against a closed index returned SUCCESS with the entry committed to the
+	 * log and silently absent from the index - the exact log/index divergence this
+	 * recovery exists to avoid, manufactured on the public write path with no error
+	 * signal. On :2863 a swallowed read reads as "already deleted" and returns
+	 * `assume ok`, turning a REJECT into an ALLOW for inbound remote deletes.
+	 *
+	 * Those callers must keep throwing: outside change delivery, a closed index is
+	 * a caller error, not a state to paper over.
+	 */
+	private getLocalIndexedContextForChange(
+		key: indexerTypes.IdKey,
+	): Promise<indexerTypes.IndexedResult<IndexedContextOnly<I>> | undefined> {
+		// Not every backing index resolves asynchronously (the native one answers
+		// synchronously, and may throw synchronously), so guard both shapes without
+		// forcing a promise on the fast path.
+		type LocalIndexedContext =
+			| indexerTypes.IndexedResult<IndexedContextOnly<I>>
+			| undefined;
+		try {
+			const result = this.getLocalIndexedContext(
+				key,
+			) as MaybePromise<LocalIndexedContext>;
+			return (
+				isPromiseLike(result)
+					? result.catch((error) => this.recoverClosedIndexContextRead(error))
+					: result
+			) as Promise<LocalIndexedContext>;
+		} catch (error) {
+			return this.recoverClosedIndexContextRead(
+				error,
+			) as unknown as Promise<LocalIndexedContext>;
+		}
+	}
+
+	/**
+	 * Deliberately narrow: only `NotStartedError`, and only once the document index
+	 * is actually closed. An indexer that breaks while the index is open still
+	 * throws. Reachable only from `getLocalIndexedContextForChange`.
+	 */
+	private recoverClosedIndexContextRead(error: unknown): undefined {
+		if (error instanceof indexerTypes.NotStartedError && this._index.closed) {
+			return undefined;
+		}
+		throw error;
+	}
+
 	private getNativeEntrySignerPublicKeys(
 		hashes: string[],
 	): Array<Uint8Array | undefined> | undefined {
@@ -2442,6 +2511,27 @@ export class Documents<
 		return this.log.recover();
 	}
 
+	/**
+	 * `Documents` owns two sub-programs: `log` (the source of changes) and
+	 * `_index` (a view derived from it, backed by an indexer). `Program` tears
+	 * children down concurrently by default, so nothing stopped
+	 * `DocumentIndex.close()` -> `indexer.stop()` from completing while a
+	 * `Log.joinRecursively()` was still in flight. That join calls `handleChanges`
+	 * with an entry it has *already* committed to the log, and `handleChanges`
+	 * reads the index (`getLocalIndexedContext`) before it writes - so the read hit
+	 * an indexer in state "closed" and threw `NotStartedError` out of a join that
+	 * had no business failing.
+	 *
+	 * Ranking the index behind the log fixes the ordering rather than the symptom:
+	 * `SharedLog.close()` drains its in-flight receives and then closes the lower
+	 * `Log`, so by the time the index is torn down no further change can be
+	 * dispatched, and every change already committed to the log has been applied to
+	 * the index. Nothing is dropped.
+	 */
+	protected terminalChildOrder(child: Program): number {
+		return child === (this._index as Program) ? 1 : 0;
+	}
+
 	private async _resolveEntry(
 		history: Entry<Operation> | string,
 		options?: {
@@ -5369,7 +5459,7 @@ export class Documents<
 								? reference.existing
 								: this.isNativeMode()
 									? this.getNativeModeIndexedContext(key) || null
-									: (await this.getLocalIndexedContext(key)) || null;
+									: (await this.getLocalIndexedContextForChange(key)) || null;
 					if (!this.strictHistory && existing) {
 						// if immutable use oldest, else use newest
 						let shouldIgnoreChange = this.immutable

package/src/search.ts

@@ -15,12 +15,6 @@ import {
 	sha256Base64Sync,
 } from "@peerbit/crypto";
 import * as types from "@peerbit/document-interface";
-import {
-	type SimpleDocumentFieldExtractionPlan,
-	type SimpleDocumentProjectionContext,
-	type SimpleDocumentProjectionPlan,
-	tryProjectDocumentIndexSimple,
-} from "./native-rust.js";
 import { CachedIndex, type QueryCacheOptions } from "@peerbit/indexer-cache";
 import * as indexerTypes from "@peerbit/indexer-interface";
 import { HashmapIndex } from "@peerbit/indexer-simple";
@@ -56,6 +50,12 @@ import type { QueryPredictor } from "./most-common-query-predictor.js";
 import MostCommonQueryPredictor, {
 	idAgnosticQueryKey,
 } from "./most-common-query-predictor.js";
+import {
+	type SimpleDocumentFieldExtractionPlan,
+	type SimpleDocumentProjectionContext,
+	type SimpleDocumentProjectionPlan,
+	tryProjectDocumentIndexSimple,
+} from "./native-rust.js";
 import { type Operation, isPutOperation } from "./operation.js";
 import { Prefetch } from "./prefetch.js";
 import type { ExtractArgs } from "./program.js";
@@ -67,13 +67,13 @@ import {
 } from "./result-shape.js";
 import { ResumableIterators } from "./resumable-iterator.js";
 import {
+	type DocumentTransformDescriptor,
+	type DocumentTransformFacts,
+	type DocumentTransformer,
 	canPrepareDocumentTransformBeforeAppend,
 	canPrepareDocumentTransformWithAppendFacts,
 	documentTransformPreservesFieldPath,
 	getDocumentTransformDescriptor,
-	type DocumentTransformFacts,
-	type DocumentTransformDescriptor,
-	type DocumentTransformer,
 } from "./transform.js";
 
 const WARNING_WHEN_ITERATING_FOR_MORE_THAN = 1e5;
@@ -1221,10 +1221,9 @@ export class DocumentIndex<
 			} catch {
 				// A native-backed head may be hollow until read from the block store.
 			}
-			return Entry.fromMultihash<Operation>(
-				this._log.log.blocks,
-				hash,
-			).catch(() => head);
+			return Entry.fromMultihash<Operation>(this._log.log.blocks, hash).catch(
+				() => head,
+			);
 		}
 		return Entry.fromMultihash<Operation>(this._log.log.blocks, hash);
 	}
@@ -1458,18 +1457,12 @@ export class DocumentIndex<
 							claimedIds = wrapped
 								.map((result) =>
 									result instanceof types.ResultValue && result.indexed
-										? indexerTypes.toId(
-												this.indexByResolver(result.indexed),
-											)
+										? indexerTypes.toId(this.indexByResolver(result.indexed))
 										: result instanceof types.ResultIndexedValue
-											? indexerTypes.toId(
-													this.indexByResolver(result.value),
-												)
+											? indexerTypes.toId(this.indexByResolver(result.value))
 											: undefined,
 								)
-								.filter(
-									(id): id is indexerTypes.IdKey => id !== undefined,
-								);
+								.filter((id): id is indexerTypes.IdKey => id !== undefined);
 						}
 					}
 					if (batches.length) {
@@ -1792,9 +1785,7 @@ export class DocumentIndex<
 	private canPrepareNativeBackboneDocumentIndexCommit(): boolean {
 		return (
 			(this.transformerIsIdentity && this.indexedTypeIsDocumentType) ||
-			canPrepareDocumentTransformBeforeAppend(
-				this.nativeTransformDescriptor,
-			)
+			canPrepareDocumentTransformBeforeAppend(this.nativeTransformDescriptor)
 		);
 	}
 
@@ -1802,9 +1793,7 @@ export class DocumentIndex<
 		return (
 			this.canPrepareNativeBackboneDocumentIndexCommit() ||
 			!!this.nativeTransformProjectionPlan ||
-			canPrepareDocumentTransformWithAppendFacts(
-				this.nativeTransformDescriptor,
-			)
+			canPrepareDocumentTransformWithAppendFacts(this.nativeTransformDescriptor)
 		);
 	}
 
@@ -1865,9 +1854,7 @@ export class DocumentIndex<
 			};
 		}
 		if (
-			!canPrepareDocumentTransformBeforeAppend(
-				this.nativeTransformDescriptor,
-			)
+			!canPrepareDocumentTransformBeforeAppend(this.nativeTransformDescriptor)
 		) {
 			return;
 		}
@@ -1950,9 +1937,7 @@ export class DocumentIndex<
 			!canPrepareDocumentTransformWithAppendFacts(
 				this.nativeTransformDescriptor,
 			) &&
-			!canPrepareDocumentTransformBeforeAppend(
-				this.nativeTransformDescriptor,
-			)
+			!canPrepareDocumentTransformBeforeAppend(this.nativeTransformDescriptor)
 		) {
 			return;
 		}
@@ -1991,9 +1976,7 @@ export class DocumentIndex<
 			!canPrepareDocumentTransformWithAppendFacts(
 				this.nativeTransformDescriptor,
 			) &&
-			!canPrepareDocumentTransformBeforeAppend(
-				this.nativeTransformDescriptor,
-			)
+			!canPrepareDocumentTransformBeforeAppend(this.nativeTransformDescriptor)
 		) {
 			return;
 		}
@@ -2488,9 +2471,7 @@ export class DocumentIndex<
 		return indexed?.id;
 	}
 
-	private getIndexedKeyByHead(
-		head: string,
-	): indexerTypes.IdKey | undefined {
+	private getIndexedKeyByHead(head: string): indexerTypes.IdKey | undefined {
 		const getIdByHead = (this.index as ContextHeadIndex<I>).getIdByContextHead;
 		return typeof getIdByHead === "function"
 			? getIdByHead.call(this.index, head)
@@ -2507,9 +2488,10 @@ export class DocumentIndex<
 		return heads.map((head) => getIdByHead.call(this.index, head));
 	}
 
-	public tryGetIdentityIndexedKeyByHead(
-		head: string,
-	): { supported: boolean; key?: indexerTypes.IdKey } {
+	public tryGetIdentityIndexedKeyByHead(head: string): {
+		supported: boolean;
+		key?: indexerTypes.IdKey;
+	} {
 		const getIdByHead = (this.index as ContextHeadIndex<I>).getIdByContextHead;
 		if (typeof getIdByHead !== "function") {
 			return { supported: false };
@@ -2849,12 +2831,18 @@ export class DocumentIndex<
 				return false;
 			}
 			return isPromiseLike(persistResult)
-				? persistResult.then(() => true, (error: unknown) => {
-						if (error instanceof indexerTypes.NotStartedError && this.closed) {
-							return true;
-						}
-						throw error;
-					})
+				? persistResult.then(
+						() => true,
+						(error: unknown) => {
+							if (
+								error instanceof indexerTypes.NotStartedError &&
+								this.closed
+							) {
+								return true;
+							}
+							throw error;
+						},
+					)
 				: true;
 		} catch (error) {
 			if (error instanceof indexerTypes.NotStartedError && this.closed) {
@@ -2974,9 +2962,7 @@ export class DocumentIndex<
 		}
 		const batchValues: Array<{
 			id: indexerTypes.IdKey;
-			encodedValueParts: NonNullable<
-				ContextualPutOptions["encodedValueParts"]
-			>;
+			encodedValueParts: NonNullable<ContextualPutOptions["encodedValueParts"]>;
 			options?: { replace?: boolean };
 		}> = [];
 		for (const item of values) {
@@ -3341,6 +3327,29 @@ export class DocumentIndex<
 		});
 	}
 
+	/**
+	 * The removal counterpart of `putWithContext`'s guard. Once this index is
+	 * closed there is nothing left to remove from, so a `NotStartedError` marks
+	 * the teardown window rather than a failure.
+	 *
+	 * This exists because the guard used to be one-legged: the read leg
+	 * (`Documents.getLocalIndexedContextForChange`) tolerated a closed index while
+	 * the remove leg did not, so a DELETE-carrying change still threw out of a
+	 * join in exactly the window the read guard was written for. A guard that
+	 * covers one direction of the same operation is worse than none, because it
+	 * reads as covered.
+	 *
+	 * Narrow in the same way as the read leg: only `NotStartedError`, and only
+	 * once this index is actually closed. An indexer that breaks while open still
+	 * throws.
+	 */
+	private recoverClosedRemoval(error: unknown): void {
+		if (error instanceof indexerTypes.NotStartedError && this.closed) {
+			return;
+		}
+		throw error;
+	}
+
 	public async delMany(keys: indexerTypes.IdKey[]): Promise<void> {
 		if (keys.length === 0) {
 			return;
@@ -3348,17 +3357,21 @@ export class DocumentIndex<
 		for (const key of keys) {
 			this.deleteResolvedCacheForKey(key);
 		}
-		const delIdsNoReturn = (this.index as ExactDeleteIndex).delIdsNoReturn;
-		if (delIdsNoReturn) {
-			await delIdsNoReturn.call(this.index, keys);
-			return;
-		}
-		const delIds = (this.index as ExactDeleteIndex).delIds;
-		if (delIds) {
-			await delIds.call(this.index, keys);
-			return;
+		try {
+			const delIdsNoReturn = (this.index as ExactDeleteIndex).delIdsNoReturn;
+			if (delIdsNoReturn) {
+				await delIdsNoReturn.call(this.index, keys);
+				return;
+			}
+			const delIds = (this.index as ExactDeleteIndex).delIds;
+			if (delIds) {
+				await delIds.call(this.index, keys);
+				return;
+			}
+			await Promise.all(keys.map((key) => this.del(key)));
+		} catch (error) {
+			this.recoverClosedRemoval(error);
 		}
-		await Promise.all(keys.map((key) => this.del(key)));
 	}
 
 	public clearResolvedCacheForKeys(keys: indexerTypes.IdKey[]): void {
@@ -3374,17 +3387,28 @@ export class DocumentIndex<
 		for (const key of keys) {
 			this.deleteResolvedCacheForKey(key);
 		}
-		const delIdsNoReturn = (this.index as ExactDeleteIndex).delIdsNoReturn;
-		if (delIdsNoReturn) {
-			const result = delIdsNoReturn.call(this.index, keys);
-			return isPromiseLike(result) ? result.then(() => undefined) : undefined;
-		}
-		const delIds = (this.index as ExactDeleteIndex).delIds;
-		if (delIds) {
-			const result = delIds.call(this.index, keys);
-			return isPromiseLike(result) ? result.then(() => undefined) : undefined;
+		// The native index answers synchronously and may throw synchronously, so
+		// both shapes need the guard - see recoverClosedRemoval.
+		const guard = (result: MaybePromise<unknown>): MaybePromise<void> =>
+			isPromiseLike(result)
+				? result.then(
+						() => undefined,
+						(error: unknown) => this.recoverClosedRemoval(error),
+					)
+				: undefined;
+		try {
+			const delIdsNoReturn = (this.index as ExactDeleteIndex).delIdsNoReturn;
+			if (delIdsNoReturn) {
+				return guard(delIdsNoReturn.call(this.index, keys));
+			}
+			const delIds = (this.index as ExactDeleteIndex).delIds;
+			if (delIds) {
+				return guard(delIds.call(this.index, keys));
+			}
+			return guard(Promise.all(keys.map((key) => this.del(key))));
+		} catch (error) {
+			return this.recoverClosedRemoval(error);
 		}
-		return Promise.all(keys.map((key) => this.del(key))).then(() => undefined);
 	}
 
 	public async getDetailed<
@@ -5457,7 +5481,7 @@ export class DocumentIndex<
 										"Failed to collect sorted results from self. " + e?.message,
 									);
 									peerBufferMap.delete(peer);
-										}),
+								}),
 						);
 					} else {
 						// Fetch remotely
@@ -5823,9 +5847,7 @@ export class DocumentIndex<
 				keepRemoteAlive
 					? [...peerBufferMap.keys()].filter((peer) => peer !== selfHash)
 					: [...peerBufferMap.entries()]
-							.filter(
-								([peer, buffer]) => peer !== selfHash && buffer.kept > 0,
-							)
+							.filter(([peer, buffer]) => peer !== selfHash && buffer.kept > 0)
 							.map(([peer]) => peer),
 			);
 			for (const peer of remoteIteratorPeersToClose) {
@@ -5859,8 +5881,9 @@ export class DocumentIndex<
 					if (activeRemotePeers.has(peer)) {
 						ids.push(remoteIteratorIds?.get(peer) ?? queryRequestCoerced.id);
 					}
-					for (const staleRemoteIteratorId of
-						staleRemoteIteratorIds.get(peer) ?? []) {
+					for (const staleRemoteIteratorId of staleRemoteIteratorIds.get(
+						peer,
+					) ?? []) {
 						if (!ids.some((id) => equals(id, staleRemoteIteratorId))) {
 							ids.push(staleRemoteIteratorId);
 						}