@peerbit/document 13.1.5 → 13.1.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/document",
-  "version": "13.1.5",
+  "version": "13.1.7",
   "description": "Document store implementation",
   "type": "module",
   "sideEffects": false,
@@ -49,7 +49,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "author": "dao.xyz",
+  "author": "Peerbit contributors",
   "license": "MIT",
   "dependencies": {
     "@chainsafe/libp2p-yamux": "^8.0.1",
@@ -59,20 +59,20 @@
     "@multiformats/multiaddr": "^13.0.1",
     "p-defer": "^4.0.0",
     "uint8arrays": "^5.1.0",
-    "@peerbit/indexer-cache": "0.2.8",
-    "@peerbit/indexer-interface": "3.0.5",
     "@peerbit/cache": "3.1.0",
-    "@peerbit/document-interface": "3.2.47",
-    "@peerbit/indexer-simple": "1.2.9",
-    "@peerbit/crypto": "3.1.2",
-    "@peerbit/log": "6.2.4",
+    "@peerbit/document-interface": "3.2.49",
+    "@peerbit/indexer-cache": "0.2.10",
+    "@peerbit/crypto": "3.1.3",
+    "@peerbit/indexer-interface": "3.0.7",
+    "@peerbit/indexer-simple": "1.2.11",
+    "@peerbit/indexer-sqlite3": "3.0.10",
+    "@peerbit/log": "6.2.6",
     "@peerbit/logger": "2.0.1",
-    "@peerbit/indexer-sqlite3": "3.0.8",
-    "@peerbit/pubsub": "5.3.0",
-    "@peerbit/program": "6.0.34",
-    "@peerbit/rpc": "6.1.2",
-    "@peerbit/shared-log": "13.2.5",
-    "@peerbit/stream-interface": "6.0.11"
+    "@peerbit/program": "6.0.36",
+    "@peerbit/pubsub": "5.3.1",
+    "@peerbit/rpc": "6.1.4",
+    "@peerbit/shared-log": "13.2.7",
+    "@peerbit/stream-interface": "6.0.12"
   },
   "optionalDependencies": {
     "@peerbit/document-rust": "0.1.1"
@@ -82,12 +82,12 @@
     "@libp2p/websockets": "^10.1.7",
     "@types/pidusage": "^2.0.5",
     "pidusage": "^4.0.1",
-    "uuid": "^10.0.0",
+    "uuid": "^11.1.1",
+    "@peerbit/native-backbone": "0.2.0",
     "@peerbit/log-rust": "1.1.2",
-    "@peerbit/native-backbone": "0.1.4",
-    "@peerbit/time": "3.0.0",
-    "@peerbit/test-utils": "3.1.5",
-    "peerbit": "5.3.5"
+    "@peerbit/test-utils": "3.1.7",
+    "@peerbit/time": "3.0.1",
+    "peerbit": "5.3.7"
   },
   "repository": {
     "type": "git",
@@ -102,7 +102,7 @@
     "clean": "aegir clean",
     "build": "aegir build --no-bundle",
     "test": "aegir test --target node",
-    "test:document-rust-core": "npm run build && PEERBIT_SHARED_LOG_RUST_CORE=1 aegir test -t node --grep '^(?!.*(uses the validated local append path|uses the independent native prepared batch path|uses native shared-log planning for replicated target-none puts|uses commit-only local puts when coordinate persistence is deferred|can delete without being replicator)).*(native conformance guard |operations basic |operations get |operations index |operations search |iterate sort |count approximate |query distribution |returnIndexed |caching |custom index |prefetch )'",
+    "test:document-rust-core": "npm run build && PEERBIT_SHARED_LOG_RUST_CORE=1 aegir test -t node --grep '^(?!.*(uses the validated local append path|uses the independent native prepared batch path|uses native shared-log planning for replicated target-none puts|uses commit-only local puts when coordinate persistence is deferred)).*(native conformance guard |operations basic |operations get |operations index |operations search |iterate sort |count approximate |query distribution |returnIndexed |caching |custom index |prefetch )' -- --forbid-pending",
     "lint": "aegir lint",
     "test:cov": "aegir test -t node --cov"
   }

package/src/program.ts

@@ -735,6 +735,7 @@ type TrustedDocumentSharedLogAppendManyResult = {
 };
 
 type TrustedDocumentSharedLog = {
+	finishNativeStrictDurableDocumentRecovery(): Promise<void>;
 	appendLocallyPrepared(
 		data: Operation,
 		options?: SharedAppendOptions<Operation>,
@@ -2418,6 +2419,9 @@ export class Documents<
 				await initializeDocumentRust();
 			}
 		}
+		await asTrustedDocumentSharedLog(
+			this.log,
+		).finishNativeStrictDurableDocumentRecovery();
 
 		this._optionCanPerformNativeFastPath = this._optionCanPerformNativePolicy
 			? createCanPerformPolicyEvaluator(

package/src/resumable-iterator.ts

@@ -11,6 +11,11 @@ import { logger as loggerFn } from "@peerbit/logger";
 const iteratorLogger = loggerFn("peerbit:document:index:iterate");
 
 export class ResumableIterators<T extends Record<string, any>> {
+	private readonly pendingMarks = new Map<
+		string,
+		Map<string, indexerTypes.IdKey>
+	>();
+
 	constructor(
 		readonly index: indexerTypes.Index<T>,
 		readonly queues = new Cache<{
@@ -22,6 +27,36 @@ export class ResumableIterators<T extends Record<string, any>> {
 		// TODO choose upper limit better
 	}
 
+	private markKey(id: indexerTypes.IdKey) {
+		return `${id.key instanceof Uint8Array ? "bytes" : typeof id.key}:${String(id.primitive)}`;
+	}
+
+	private retainMarks(id: string, ids: Iterable<indexerTypes.IdKey>) {
+		let retained = this.pendingMarks.get(id);
+		for (const mark of ids) {
+			if (!retained) {
+				retained = new Map();
+				this.pendingMarks.set(id, retained);
+			}
+			retained.set(this.markKey(mark), mark);
+		}
+	}
+
+	private async applyPendingMarks(
+		id: string,
+		iterator: indexerTypes.IndexIterator<T, undefined>,
+	) {
+		while (true) {
+			const retained = this.pendingMarks.get(id);
+			if (!retained?.size) {
+				this.pendingMarks.delete(id);
+				return;
+			}
+			this.pendingMarks.delete(id);
+			await iterator.markYielded?.(retained.values());
+		}
+	}
+
 	async iterateAndFetch(
 		request: SearchRequest | SearchRequestIndexed | IterationRequest,
 		options?: { keepAlive?: boolean },
@@ -31,29 +66,32 @@ export class ResumableIterators<T extends Record<string, any>> {
 			fetch: request.fetch,
 			keepAlive: Boolean(options?.keepAlive),
 		});
-		const iterator = this.index.iterate(request);
-		const firstResult = await iterator.next(request.fetch);
 		const keepAlive = options?.keepAlive === true;
-		if (keepAlive || iterator.done() !== true) {
-			const cachedIterator = {
-				iterator,
-				request,
+		const iterator = this.index.iterate(request);
+		this.queues.add(request.idString, { iterator, request, keepAlive });
+		try {
+			await this.applyPendingMarks(request.idString, iterator);
+			const firstResult = await iterator.next(request.fetch);
+			if (!keepAlive && iterator.done() === true) {
+				this.queues.del(request.idString);
+				this.pendingMarks.delete(request.idString);
+			}
+			iteratorLogger("iterate:queued", {
+				id: request.idString,
 				keepAlive,
-			};
-			this.queues.add(request.idString, cachedIterator);
+				done: iterator.done() === true,
+				batch: firstResult.length,
+			});
+			/* console.debug(
+				"[ResumableIterators] iterateAndFetch",
+				request.idString,
+				{ keepAlive },
+			); */
+			return firstResult;
+		} catch (error) {
+			this.clear(request.idString);
+			throw error;
 		}
-		iteratorLogger("iterate:queued", {
-			id: request.idString,
-			keepAlive,
-			done: iterator.done() === true,
-			batch: firstResult.length,
-		});
-		/* console.debug(
-			"[ResumableIterators] iterateAndFetch",
-			request.idString,
-			{ keepAlive },
-		); */
-		return firstResult;
 	}
 
 	async next(
@@ -138,6 +176,7 @@ export class ResumableIterators<T extends Record<string, any>> {
 			}
 		}
 		this.queues.del(id);
+		this.pendingMarks.delete(id);
 	}
 
 	async clearAll() {
@@ -155,6 +194,7 @@ export class ResumableIterators<T extends Record<string, any>> {
 			),
 		);
 		this.queues.clear();
+		this.pendingMarks.clear();
 	}
 
 	has(id: string) {
@@ -179,4 +219,12 @@ export class ResumableIterators<T extends Record<string, any>> {
 		});
 		return pending;
 	}
+
+	async markYielded(id: string, ids: Iterable<indexerTypes.IdKey>) {
+		this.retainMarks(id, ids);
+		const iterator = this.queues.get(id);
+		if (iterator) {
+			await this.applyPendingMarks(id, iterator.iterator);
+		}
+	}
 }