@peerbit/document 15.0.11 → 15.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/document",
-  "version": "15.0.11",
+  "version": "15.0.13",
   "description": "Document store implementation",
   "type": "module",
   "sideEffects": false,
@@ -61,20 +61,20 @@
     "p-defer": "^4.0.0",
     "uint8arrays": "^5.1.0",
     "@peerbit/cache": "3.1.1",
+    "@peerbit/document-interface": "3.2.66",
     "@peerbit/crypto": "3.1.6",
-    "@peerbit/document-interface": "3.2.64",
-    "@peerbit/indexer-cache": "0.2.14",
-    "@peerbit/indexer-interface": "3.0.11",
-    "@peerbit/indexer-simple": "1.2.15",
-    "@peerbit/indexer-sqlite3": "3.0.16",
-    "@peerbit/log": "6.2.21",
-    "@peerbit/program": "6.0.53",
+    "@peerbit/indexer-cache": "0.2.15",
+    "@peerbit/indexer-interface": "3.0.12",
+    "@peerbit/indexer-simple": "1.2.16",
+    "@peerbit/indexer-sqlite3": "3.0.18",
+    "@peerbit/log": "6.2.23",
     "@peerbit/logger": "2.0.2",
+    "@peerbit/program": "6.0.54",
     "@peerbit/pubsub": "5.4.2",
-    "@peerbit/rpc": "6.1.21",
+    "@peerbit/shared-log": "16.0.13",
+    "@peerbit/rpc": "6.1.22",
     "@peerbit/stream-interface": "6.0.16",
-    "@peerbit/time": "3.0.1",
-    "@peerbit/shared-log": "16.0.11"
+    "@peerbit/time": "3.0.1"
   },
   "optionalDependencies": {
     "@peerbit/document-rust": "0.1.1"
@@ -87,8 +87,8 @@
     "uuid": "^11.1.1",
     "@peerbit/log-rust": "1.1.4",
     "@peerbit/native-backbone": "0.2.10",
-    "@peerbit/test-utils": "3.1.31",
-    "peerbit": "5.3.31"
+    "@peerbit/test-utils": "3.1.33",
+    "peerbit": "5.3.33"
   },
   "repository": {
     "type": "git",
@@ -102,6 +102,8 @@
   "scripts": {
     "clean": "aegir clean",
     "build": "aegir build --no-bundle",
+    "bench:ordered-remote-projection": "node --loader ts-node/esm ./benchmark/ordered-remote-projection.ts",
+    "bench:shared-fs-cold-join": "node --loader ts-node/esm ./benchmark/shared-fs-cold-join.ts",
     "test": "aegir test --target node",
     "test:document-rust-core": "npm run build && PEERBIT_SHARED_LOG_RUST_CORE=1 aegir test -t node --grep '^(?!.*(uses the validated local append path|uses the independent native prepared batch path|uses native shared-log planning for replicated target-none puts|uses commit-only local puts when coordinate persistence is deferred)).*(native conformance guard |operations basic |operations get |operations index |operations search |iterate sort |count approximate |query distribution |returnIndexed |caching |custom index |prefetch )' -- --forbid-pending",
     "lint": "aegir lint",

package/src/program.ts

@@ -74,6 +74,7 @@ import {
 	type CanRead,
 	type CanSearch,
 	DocumentIndex,
+	type DocumentIndexWriteSession,
 	type GetOptions,
 	INDEX_CONTEXT_SHAPE,
 	type IndexedContextOnly,
@@ -2025,6 +2026,7 @@ export class Documents<
 	 */
 	private getLocalIndexedContextForChange(
 		key: indexerTypes.IdKey,
+		orderedSession?: DocumentIndexWriteSession<I>,
 	): Promise<indexerTypes.IndexedResult<IndexedContextOnly<I>> | undefined> {
 		// Not every backing index resolves asynchronously (the native one answers
 		// synchronously, and may throw synchronously), so guard both shapes without
@@ -2033,9 +2035,9 @@ export class Documents<
 			| indexerTypes.IndexedResult<IndexedContextOnly<I>>
 			| undefined;
 		try {
-			const result = this.getLocalIndexedContext(
-				key,
-			) as MaybePromise<LocalIndexedContext>;
+			const result = (orderedSession
+				? orderedSession.get(key, { shape: INDEX_CONTEXT_SHAPE })
+				: this.getLocalIndexedContext(key)) as MaybePromise<LocalIndexedContext>;
 			return (
 				isPromiseLike(result)
 					? result.catch((error) => this.recoverClosedIndexContextRead(error))
@@ -5342,6 +5344,24 @@ export class Documents<
 		reference?: PutChangeReference<T, I>,
 	): Promise<void> {
 		logger.trace("handleChanges called", change);
+		if (
+			reference == null &&
+			change.added.length >= 2 &&
+			!this.isNativeMode() &&
+			this._index.canUseOrderedWriteSession()
+		) {
+			return this._index.withOrderedWriteSession((orderedSession) =>
+				this.handleChangesInOrder(change, reference, orderedSession),
+			);
+		}
+		return this.handleChangesInOrder(change, reference);
+	}
+
+	private async handleChangesInOrder(
+		change: Change<Operation>,
+		reference?: PutChangeReference<T, I>,
+		orderedSession?: DocumentIndexWriteSession<I>,
+	): Promise<void> {
 		const isAppendOperation =
 			change?.added.length === 1 ? !!change.added[0] : false;
 		const removedSet = new Map<string, ShallowOrFullEntry<Operation>>();
@@ -5382,6 +5402,7 @@ export class Documents<
 		let modified: Set<string | number | bigint> = new Set();
 		for (const item of sortedEntries) {
 			if (!item) {
+				await orderedSession?.flush();
 				continue;
 			}
 
@@ -5394,6 +5415,7 @@ export class Documents<
 					? reference.operation
 					: await this.getAppendOperation(item);
 				if (!payload) {
+					await orderedSession?.flush();
 					continue;
 				}
 
@@ -5448,6 +5470,9 @@ export class Documents<
 
 					// document is already updated with more recent entry
 					if (modified.has(key.primitive)) {
+						// Duplicate ids deliberately retain the scalar boundary until
+						// their callback and event semantics are proven independently.
+						await orderedSession?.flush();
 						continue;
 					}
 
@@ -5459,7 +5484,10 @@ export class Documents<
 								? reference.existing
 								: this.isNativeMode()
 									? this.getNativeModeIndexedContext(key) || null
-									: (await this.getLocalIndexedContextForChange(key)) || null;
+									: (await this.getLocalIndexedContextForChange(
+											key,
+											orderedSession,
+										)) || null;
 					if (!this.strictHistory && existing) {
 						// if immutable use oldest, else use newest
 						let shouldIgnoreChange = this.immutable
@@ -5468,10 +5496,19 @@ export class Documents<
 							: existing.value.__context.modified >
 								item.meta.clock.timestamp.wallTime;
 						if (shouldIgnoreChange) {
+							await orderedSession?.flush();
 							continue;
 						}
 					}
 
+					const useOrderedWrite =
+						orderedSession != null &&
+						existing == null &&
+						!(value instanceof Program);
+					if (!useOrderedWrite) {
+						await orderedSession?.flush();
+					}
+
 					// Program specific
 					if (value instanceof Program) {
 						// if replicator, then open
@@ -5497,6 +5534,7 @@ export class Documents<
 						key,
 						item,
 						existing,
+						useOrderedWrite ? orderedSession : undefined,
 					);
 					documentsChanged?.added.push(
 						coerceWithIndexed(coerceWithContext(value, context), indexable),
@@ -5508,6 +5546,7 @@ export class Documents<
 					isPutOperation(payload) ||
 					removedSet.has(item.hash)
 				) {
+					await orderedSession?.flush();
 					if (
 						!documentsChanged &&
 						(await this.collectRemovedPutChangeFromNativeId(payload, modified))
@@ -5525,12 +5564,14 @@ export class Documents<
 				}
 			} catch (error) {
 				if (error instanceof AccessError) {
+					await orderedSession?.flush();
 					continue;
 				}
 				throw error;
 			}
 		}
 
+		await orderedSession?.flush();
 		if (canRemoveByIndexedHead && change.removed.length > 0) {
 			const handled = await this.collectRemovedDocumentChangesFromIndexedHeads(
 				change.removed,

package/src/search.ts

@@ -739,6 +739,9 @@ export type WithContext<I> = {
 	__context: types.Context;
 } & I;
 
+export type DocumentIndexWriteSession<I extends Record<string, any>> =
+	indexerTypes.OrderedIndexWriteSession<WithContext<I>>;
+
 export const INDEX_CONTEXT_SHAPE = {
 	__context: {
 		created: true,
@@ -1135,6 +1138,23 @@ export class DocumentIndex<
 		return this._valueEncoding;
 	}
 
+	public canUseOrderedWriteSession(): boolean {
+		return (
+			!this.isProgramValued &&
+			typeof this.index.withOrderedWriteSession === "function"
+		);
+	}
+
+	public async withOrderedWriteSession<R>(
+		operation: (session: DocumentIndexWriteSession<I>) => MaybePromise<R>,
+	): Promise<R> {
+		const withSession = this.index.withOrderedWriteSession;
+		if (!withSession) {
+			throw new Error("Backing index does not support ordered write sessions");
+		}
+		return withSession.call(this.index, operation) as MaybePromise<R>;
+	}
+
 	private ensurePrefetchAccumulator() {
 		if (!this._prefetch) {
 			this._prefetch = {
@@ -3090,10 +3110,11 @@ export class DocumentIndex<
 			| indexerTypes.IndexedResult<IndexedContextOnly<I>>
 			| null
 			| undefined,
+		orderedSession?: DocumentIndexWriteSession<I>,
 	): Promise<{ context: types.Context; indexable: I }> {
 		const existingDefined =
 			existing === undefined
-				? await this.index.get(id, {
+				? await (orderedSession ?? this.index).get(id, {
 						shape: INDEX_CONTEXT_SHAPE,
 					})
 				: existing;
@@ -3106,10 +3127,16 @@ export class DocumentIndex<
 			gid: entry.meta.gid,
 			size: entry.payload.byteLength,
 		});
-		return this.putWithContext(value, id, context, {
-			replace: existingDefined != null,
-			transformFacts: { entryPublicKeys: entry.publicKeys },
-		});
+		return this.putWithContext(
+			value,
+			id,
+			context,
+			{
+				replace: existingDefined != null,
+				transformFacts: { entryPublicKeys: entry.publicKeys },
+			},
+			orderedSession,
+		);
 	}
 
 	public async putWithContext(
@@ -3117,6 +3144,7 @@ export class DocumentIndex<
 		id: indexerTypes.IdKey,
 		context: types.Context,
 		options?: ContextualPutOptions,
+		orderedSession?: DocumentIndexWriteSession<I>,
 	): Promise<{ context: types.Context; indexable: I }> {
 		const idString = id.primitive;
 		this.cacheResolvedValue(idString, value);
@@ -3129,10 +3157,17 @@ export class DocumentIndex<
 		coerceWithContext(value, context);
 
 		try {
-			const contextualPut = this.transformerIsIdentity
-				? (this.index as ContextualIndexPut<I>).putWithContext
-				: undefined;
-			if (contextualPut) {
+			const contextualPut =
+				!orderedSession && this.transformerIsIdentity
+					? (this.index as ContextualIndexPut<I>).putWithContext
+					: undefined;
+			if (orderedSession) {
+				await orderedSession.put(
+					new this.wrappedIndexedType(valueToIndex, context),
+					id,
+					stripEncodedValue(options),
+				);
+			} else if (contextualPut) {
 				const encodedValueParts = this.encodeContextualIndexedValueParts(
 					options?.encodedValue,
 					context,