@verdant-web/server 3.3.3 → 3.3.4-alpha.1

package/src/ServerLibrary.ts

@@ -13,7 +13,6 @@ import {
 	applyOperations,
 	isFileRef,
 	isObjectRef,
-	omit,
 	rewriteAuthzOriginator,
 } from '@verdant-web/common';
 import { MessageSender } from './MessageSender.js';
@@ -26,6 +25,7 @@ import {
 	LibraryInfo,
 	StoredDocumentBaseline,
 	StoredOperation,
+	StoredReplicaInfo,
 } from './types.js';
 
 export type ServerLibraryEvents = {
@@ -171,13 +171,27 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 			message.operations,
 		);
 
-		// TODO: can we defer this and rebroadcast in parallel?
 		// insert patches into history
-		await this.storage.operations.insertAll(
+
+		// first, record the replica's serverOrder before insertion.
+		const replicaInfo = await this.storage.replicas.getOrCreate(
+			info.libraryId,
+			message.replicaId,
+			{
+				userId: info.userId,
+				type: info.type,
+			},
+		);
+		const newServerOrder = await this.storage.operations.insertAll(
 			info.libraryId,
 			message.replicaId,
 			message.operations,
 		);
+		this.preemptiveUpdateServerOrder(
+			replicaInfo.replicaInfo,
+			newServerOrder,
+			message.operations.length,
+		);
 
 		this.enqueueRebase(info.libraryId);
 
@@ -196,6 +210,35 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 		this.emit(`changes`, info, message.operations, []);
 	};
 
+	/**
+	 * If a replica inserts operations when it's already
+	 * up to date, we can preemptively update its ackedServerOrder
+	 * to avoid a round trip.
+	 */
+	private preemptiveUpdateServerOrder = async (
+		replicaInfo: StoredReplicaInfo,
+		newServerOrder: number,
+		insertedOperationsCount: number,
+	) => {
+		if (
+			newServerOrder - replicaInfo.ackedServerOrder ===
+			insertedOperationsCount
+		) {
+			await this.storage.replicas.updateAckedServerOrder(
+				replicaInfo.libraryId,
+				replicaInfo.id,
+				newServerOrder,
+			);
+			this.log(
+				'debug',
+				'Preemptively updated ackedServerOrder for',
+				replicaInfo.id,
+				'to',
+				newServerOrder,
+			);
+		}
+	};
+
 	private rebroadcastOperations = async (
 		libraryId: string,
 		clientKey: string,
@@ -256,13 +299,26 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 		const { status, replicaInfo: clientReplicaInfo } =
 			await this.storage.replicas.getOrCreate(info.libraryId, replicaId, info);
 
-		const changesSince =
-			status === 'existing' ? clientReplicaInfo.ackedLogicalTime : null;
 		this.log(
 			'info',
-			`Sync from ${replicaId} (user: ${info.userId}) [ackedLogicalTime: ${changesSince}, ackedServerOrder: ${clientReplicaInfo.ackedServerOrder}, status: ${status}}]`,
+			`Sync from ${replicaId} (user: ${info.userId}) [ackedServerOrder: ${clientReplicaInfo.ackedServerOrder}, status: ${status}}]`,
 		);
 
+		// for truant replicas -- if their ackedServerOrder is up-to-date, that's fine,
+		// restore them.
+		let overrideTruant = false;
+		if (status === 'truant') {
+			const latestServerOrder =
+				await this.storage.operations.getLatestServerOrder(info.libraryId);
+			if (clientReplicaInfo.ackedServerOrder >= latestServerOrder) {
+				overrideTruant = true;
+				this.log(
+					'debug',
+					'Overriding truant reset; truant replica has up-to-date server order',
+				);
+			}
+		}
+
 		// lookup operations after the last ack the client gave us
 		const ops = await this.storage.operations.getAfterServerOrder(
 			info.libraryId,
@@ -272,7 +328,8 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 		// new, unseen replicas should reset their existing storage
 		// to that of the server when joining a library.
 		// truant replicas should also reset their storage.
-		const replicaShouldReset = !!message.resyncAll || status !== 'existing';
+		const replicaShouldReset =
+			!!message.resyncAll || (!overrideTruant && status !== 'existing');
 
 		// only new replicas need baselines. by definition, a rebase only
 		// happens when all active replicas agree on history. every client
@@ -284,14 +341,17 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 
 		// We detect that a library is new by checking if it has any history.
 		// If there are no existing operations or baselines (and the requested
-		// history timerange was "everything", i.e. "from null"), then this is
+		// history timerange was "everything", i.e. "from server order 0"), then this is
 		// a fresh library which should receive the first client's history as its
 		// own. Otherwise, if the client requested a reset, we should send them
 		// the full history of the library.
 		// the definition of this field could be improved to be more explicit
 		// and not rely on seemingly unrelated data.
 		const isEmptyLibrary =
-			changesSince === null && ops.length === 0 && baselines.length === 0;
+			clientReplicaInfo.ackedServerOrder === 0 &&
+			ops.length === 0 &&
+			baselines.length === 0;
+
 		if (isEmptyLibrary) {
 			this.log('info', 'Received sync from new library', replicaId);
 
@@ -305,10 +365,12 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 		// don't reset replica if library is empty...
 		let overwriteReplicaData = replicaShouldReset && !isEmptyLibrary;
 
+		// EARLY RETURN --
 		// if the local library is empty and the replica is new to us,
 		// but the replica is providing a "since" timestamp, this
 		// suggests the local data is incomplete or gone. we request
-		// this replica should respond with a full history.
+		// this replica should respond with a full history. We will retry
+		// sync after replica has updated us with full history.
 		if (isEmptyLibrary && status === 'new' && message.since !== null) {
 			this.log(
 				'info',
@@ -360,11 +422,16 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 				message.operations.length,
 				'operations',
 			);
-			await this.storage.operations.insertAll(
+			const newServerOrder = await this.storage.operations.insertAll(
 				info.libraryId,
 				replicaId,
 				message.operations,
 			);
+			this.preemptiveUpdateServerOrder(
+				clientReplicaInfo,
+				newServerOrder,
+				message.operations.length,
+			);
 			await this.storage.replicas.updateAcknowledgedLogicalTime(
 				info.libraryId,
 				replicaId,
@@ -392,7 +459,13 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 		}
 
 		// create the nonce by encoding the server order of the last operation
-		const ackThisNonce = this.createAckNonce(ops);
+		const serverOrderToAck = ops.length
+			? ops[ops.length - 1].serverOrder
+			: undefined;
+		const ackThisNonce =
+			serverOrderToAck !== undefined
+				? this.createAckNonce(serverOrderToAck)
+				: undefined;
 
 		// respond to client
 
@@ -424,12 +497,8 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 		});
 	};
 
-	private createAckNonce = (ops: StoredOperation[]): string | undefined => {
-		return ops.length
-			? Buffer.from(JSON.stringify(ops[ops.length - 1].serverOrder)).toString(
-					'base64',
-			  )
-			: undefined;
+	private createAckNonce = (serverOrder: number): string | undefined => {
+		return Buffer.from(JSON.stringify(serverOrder)).toString('base64');
 	};
 
 	private handleAck = async (
@@ -656,8 +725,9 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 				this.log('debug', 'Database not open, skipping cleanup');
 				return;
 			}
-			const pendingDelete =
-				await this.storage.fileMetadata.getPendingDelete(libraryId);
+			const pendingDelete = await this.storage.fileMetadata.getPendingDelete(
+				libraryId,
+			);
 			if (pendingDelete.length > 0) {
 				if (!this.fileStorage) {
 					throw new Error('File storage not configured, cannot delete files');
@@ -742,8 +812,9 @@ export class ServerLibrary extends EventSubscriber<ServerLibraryEvents> {
 		const data = {
 			id: libraryId,
 			replicas,
-			latestServerOrder:
-				await this.storage.operations.getLatestServerOrder(libraryId),
+			latestServerOrder: await this.storage.operations.getLatestServerOrder(
+				libraryId,
+			),
 			operationsCount: await this.storage.operations.getCount(libraryId),
 			baselinesCount: await this.storage.baselines.getCount(libraryId),
 			globalAck: (await this.storage.replicas.getGlobalAck(libraryId)) ?? null,

package/src/storage/Storage.ts

@@ -69,7 +69,7 @@ export interface OperationStorage {
 		libraryId: string,
 		replicaId: string,
 		operations: Operation[],
-	): Promise<void>;
+	): Promise<number>;
 	deleteAll(libraryId: string): Promise<void>;
 	delete(libraryId: string, operations: Operation[]): Promise<void>;
 }

package/src/storage/sql/SqlOperations.ts

@@ -93,20 +93,21 @@ export class SqlOperations implements OperationStorage {
 		libraryId: string,
 		replicaId: string,
 		operations: Operation[],
-	): Promise<void> => {
+	): Promise<number> => {
 		// inserts all operations and updates server order
 		// FIXME: this whole thing is kinda sus
-		await this.db.transaction().execute(async (tx): Promise<void> => {
+		return await this.db.transaction().execute(async (tx): Promise<number> => {
 			let orderResult = await tx
 				.selectFrom('OperationHistory')
-				.select(({ fn, val }) =>
-					fn.coalesce(fn.max<number>('serverOrder'), val(0)).as('serverOrder'),
-				)
+				.select('serverOrder')
 				.where('libraryId', '=', libraryId)
+				.orderBy('serverOrder', 'desc')
+				.limit(1)
 				.executeTakeFirst();
 			let currentServerOrder = orderResult?.serverOrder ?? 0;
 			for (const item of operations) {
-				await tx
+				// utilizing returned serverOrder accommodates for conflicts
+				const result = await tx
 					.insertInto('OperationHistory')
 					.values({
 						libraryId,
@@ -114,7 +115,7 @@ export class SqlOperations implements OperationStorage {
 						data: JSON.stringify(item.data),
 						timestamp: item.timestamp,
 						replicaId,
-						serverOrder: ++currentServerOrder,
+						serverOrder: currentServerOrder + 1,
 						authz: item.authz,
 					})
 					.onConflict((cb) =>
@@ -122,8 +123,16 @@ export class SqlOperations implements OperationStorage {
 							.columns(['libraryId', 'replicaId', 'oid', 'timestamp'])
 							.doNothing(),
 					)
-					.execute();
+					.returning('serverOrder')
+					.executeTakeFirst();
+				if (result) {
+					currentServerOrder = result.serverOrder;
+				} else {
+					// on conflict, nothing is returned.
+					// this would mean an operation was synced twice
+				}
 			}
+			return currentServerOrder;
 		});
 	};
 

package/src/storage/sqlShard/SqlOperations.ts

@@ -1,15 +1,11 @@
 import { Operation } from '@verdant-web/common';
 import { StoredOperation } from '../../types.js';
 import { OperationStorage } from '../Storage.js';
-import { Kysely } from 'kysely';
-import { Database, OperationHistoryRow } from './tables.js';
+import { OperationHistoryRow } from './tables.js';
 import { Databases } from './Databases.js';
 
 export class SqlOperations implements OperationStorage {
-	constructor(
-		private dbs: Databases,
-		private dialect: 'postgres' | 'sqlite',
-	) {}
+	constructor(private dbs: Databases, private dialect: 'postgres' | 'sqlite') {}
 
 	private hydrate = (row: OperationHistoryRow) => {
 		// avoiding extra allocation from .map by type-asserting here
@@ -66,16 +62,17 @@ export class SqlOperations implements OperationStorage {
 
 	getLatestServerOrder = async (libraryId: string): Promise<number> => {
 		const db = await this.dbs.get(libraryId);
-		return (
-			(
-				await db
-					.selectFrom('OperationHistory')
-					.orderBy('serverOrder', 'desc')
-					.select('serverOrder')
-					.limit(1)
-					.executeTakeFirst()
-			)?.serverOrder ?? 0
-		);
+		const result = await db
+			.selectFrom('OperationHistory')
+			.orderBy('serverOrder', 'desc')
+			.select('serverOrder')
+			.limit(1)
+			.executeTakeFirst();
+		if (result) {
+			return result.serverOrder;
+		}
+
+		return 0;
 	};
 
 	getCount = async (libraryId: string): Promise<number> => {
@@ -94,34 +91,44 @@ export class SqlOperations implements OperationStorage {
 		libraryId: string,
 		replicaId: string,
 		operations: Operation[],
-	): Promise<void> => {
+	): Promise<number> => {
 		const db = await this.dbs.get(libraryId);
 		// inserts all operations and updates server order
 		// FIXME: this whole thing is kinda sus
-		await db.transaction().execute(async (tx): Promise<void> => {
+		return await db.transaction().execute(async (tx): Promise<number> => {
 			let orderResult = await tx
 				.selectFrom('OperationHistory')
-				.select(({ fn, val }) =>
-					fn.coalesce(fn.max<number>('serverOrder'), val(0)).as('serverOrder'),
-				)
+				.select('serverOrder')
+				.orderBy('serverOrder', 'desc')
+				.limit(1)
 				.executeTakeFirst();
 			let currentServerOrder = orderResult?.serverOrder ?? 0;
 			for (const item of operations) {
-				await tx
+				// utilizing returned serverOrder accommodates for conflicts
+				const result = await tx
 					.insertInto('OperationHistory')
 					.values({
 						oid: item.oid,
 						data: JSON.stringify(item.data),
 						timestamp: item.timestamp,
 						replicaId,
-						serverOrder: ++currentServerOrder,
+						serverOrder: currentServerOrder + 1,
 						authz: item.authz,
 					})
 					.onConflict((cb) =>
 						cb.columns(['replicaId', 'oid', 'timestamp']).doNothing(),
 					)
-					.execute();
+					.returning('serverOrder')
+					.executeTakeFirst();
+				if (result) {
+					currentServerOrder = result.serverOrder;
+				} else {
+					// on conflict, nothing is returned.
+					// this would mean an operation was synced twice.
+				}
 			}
+
+			return currentServerOrder;
 		});
 	};