@verdant-web/server 1.7.8

package/src/OperationHistory.ts

@@ -0,0 +1,206 @@
+import { Operation } from '@verdant-web/common';
+import { Database } from 'better-sqlite3';
+import { OperationSpec } from './types.js';
+
+type StoredOperationHistoryItem = Omit<Operation, 'data'> & {
+	data: string;
+	libraryId: string;
+	replicaId: string;
+	serverOrder: number;
+};
+
+export class OperationHistory {
+	constructor(private db: Database) {}
+
+	private hydratePatch = ({
+		libraryId: _,
+		data,
+		...rest
+	}: StoredOperationHistoryItem): OperationSpec => {
+		return {
+			...rest,
+			data: JSON.parse(data),
+		};
+	};
+
+	getAllFor = (libraryId: string, oid: string) => {
+		return this.db
+			.prepare(
+				`
+      SELECT * FROM OperationHistory
+      WHERE libraryId = ? AND oid = ?
+      ORDER BY timestamp ASC
+    `,
+			)
+			.all(libraryId, oid)
+			.map(this.hydratePatch);
+	};
+
+	getEarliest = (libraryId: string, count: number) => {
+		return this.db
+			.prepare(
+				`
+      SELECT * FROM OperationHistory
+      WHERE libraryId = ?
+      ORDER BY timestamp ASC
+      LIMIT ?
+    `,
+			)
+			.all(libraryId, count)
+			.map(this.hydratePatch);
+	};
+
+	getAfter = (libraryId: string, timestamp: string | null = null) => {
+		if (timestamp === null) {
+			return this.db
+				.prepare(
+					`
+					SELECT * FROM OperationHistory
+					WHERE libraryId = ?
+					ORDER BY timestamp ASC
+				`,
+				)
+				.all(libraryId)
+				.map(this.hydratePatch);
+		}
+		return this.db
+			.prepare(
+				`
+      SELECT * FROM OperationHistory
+      WHERE libraryId = ? AND timestamp > ?
+      ORDER BY timestamp ASC
+    `,
+			)
+			.all(libraryId, timestamp)
+			.map(this.hydratePatch);
+	};
+
+	/**
+	 * Returns all operations before the given timestamp
+	 * in ascending chronological order
+	 */
+	getBefore = (libraryId: string, before: string) => {
+		return this.db
+			.prepare(
+				`
+			SELECT * FROM OperationHistory
+			WHERE libraryId = ? AND timestamp < ?
+			ORDER BY timestamp ASC
+		`,
+			)
+			.all(libraryId, before)
+			.map(this.hydratePatch);
+	};
+
+	/**
+	 * Returns all operations after a given server order. Useful
+	 * for retrieving operations that were created after a client
+	 * last synced.
+	 */
+	getFromServerOrder = (libraryId: string, serverOrder: number) => {
+		const result = this.db
+			.prepare(
+				`
+			SELECT * FROM OperationHistory
+			WHERE libraryId = ? AND serverOrder > ?
+		`,
+			)
+			.all(libraryId, serverOrder);
+
+		return (result || []).map(this.hydratePatch);
+	};
+
+	/**
+	 * Adds a new operation or replaces an existing one.
+	 * The server order will be set to the current max + 1
+	 */
+	insert = (libraryId: string, replicaId: string, item: Operation) => {
+		return this.db
+			.prepare(
+				`
+      INSERT OR REPLACE INTO OperationHistory (libraryId, oid, data, timestamp, replicaId, serverOrder)
+      VALUES (?, ?, ?, ?, ?, (SELECT COALESCE(MAX(serverOrder), 0) FROM OperationHistory WHERE libraryId = ?) + 1)
+    `,
+			)
+			.run(
+				libraryId,
+				item.oid,
+				JSON.stringify(item.data),
+				item.timestamp,
+				replicaId,
+			);
+	};
+
+	/**
+	 * Inserts all operations and updates the server
+	 * order for this library
+	 */
+	insertAll = async (
+		libraryId: string,
+		replicaId: string,
+		items: Operation[],
+	) => {
+		const tx = this.db.transaction(() => {
+			let orderResult = this.db
+				.prepare(
+					`
+					SELECT COALESCE(MAX(serverOrder), 0) AS serverOrder FROM OperationHistory WHERE libraryId = ?
+				`,
+				)
+				.get(libraryId);
+			let currentServerOrder = orderResult.serverOrder || 0;
+			const insertStatement = this.db.prepare(
+				`
+				INSERT OR REPLACE INTO OperationHistory (libraryId, oid, data, timestamp, replicaId, serverOrder)
+				VALUES (?, ?, ?, ?, ?, ?)
+				`,
+			);
+			for (const item of items) {
+				insertStatement.run(
+					libraryId,
+					item.oid,
+					JSON.stringify(item.data),
+					item.timestamp,
+					replicaId,
+					++currentServerOrder,
+				);
+			}
+		});
+
+		tx();
+	};
+
+	dropAll = async (
+		libraryId: string,
+		items: Pick<OperationSpec, 'replicaId' | 'oid' | 'timestamp'>[],
+	) => {
+		const deleteStatement = this.db.prepare(
+			`
+			DELETE FROM OperationHistory
+			WHERE libraryId = ? AND replicaId = ? AND oid = ? AND timestamp = ?
+			`,
+		);
+
+		this.db.transaction(() => {
+			for (const item of items) {
+				deleteStatement.run(
+					libraryId,
+					item.replicaId,
+					item.oid,
+					item.timestamp,
+				);
+			}
+		})();
+	};
+
+	deleteAll = (libraryId: string) => {
+		this.db
+			.prepare(
+				`
+			DELETE FROM OperationHistory
+			WHERE libraryId = ?
+			`,
+			)
+			.run(libraryId);
+	};
+}

package/src/Presence.ts

@@ -0,0 +1,63 @@
+import { UserInfo } from '@verdant-web/common';
+import EventEmitter from 'events';
+
+/**
+ * Stores client presence in-memory for connected
+ * clients
+ */
+export class Presence extends EventEmitter {
+	private presences: Map<
+		string,
+		{
+			presences: Record<string, UserInfo<any, any>>;
+			replicaToUser: Record<string, string>;
+			userToReplica: Record<string, Set<string>>;
+		}
+	> = new Map();
+
+	private getLibrary = (libraryId: string) => {
+		let map = this.presences.get(libraryId);
+		if (!map) {
+			map = {
+				presences: {},
+				replicaToUser: {},
+				userToReplica: {},
+			};
+			this.presences.set(libraryId, map);
+		}
+		return map;
+	};
+
+	set = (libraryId: string, userId: string, presence: UserInfo<any, any>) => {
+		const lib = this.getLibrary(libraryId);
+		lib.presences[userId] = presence;
+		lib.replicaToUser[presence.replicaId] = userId;
+		lib.userToReplica[userId] = lib.userToReplica[userId] || new Set<string>();
+		lib.userToReplica[userId].add(presence.replicaId);
+	};
+
+	removeReplica = (libraryId: string, replicaId: string) => {
+		const lib = this.getLibrary(libraryId);
+		const userId = lib.replicaToUser[replicaId];
+		if (!userId) return;
+
+		lib.userToReplica[userId].delete(replicaId);
+		if (lib.userToReplica[userId].size === 0) {
+			delete lib.presences[userId];
+			this.emit('lost', libraryId, replicaId, userId);
+
+			// memory cleanup
+			if (Object.keys(lib.presences).length === 0) {
+				this.clear(libraryId);
+			}
+		}
+	};
+
+	all = (libraryId: string) => {
+		return this.presences.get(libraryId)?.presences || {};
+	};
+
+	clear = (libraryId: string) => {
+		this.presences.delete(libraryId);
+	};
+}

package/src/Profiles.ts

@@ -0,0 +1,21 @@
+export interface UserProfiles<Profile> {
+	get(userId: string): Promise<Profile>;
+}
+
+export class UserProfileLoader<Profile> {
+	private cache = new Map<string, Promise<Profile>>();
+
+	constructor(private source: UserProfiles<Profile>) {}
+
+	get = async (userId: string) => {
+		if (!this.cache.has(userId)) {
+			this.cache.set(userId, this.source.get(userId));
+		}
+
+		return this.cache.get(userId)!;
+	};
+
+	evict = (userId: string) => {
+		this.cache.delete(userId);
+	};
+}

package/src/ReplicaKeepaliveTimers.ts

@@ -0,0 +1,34 @@
+import { EventSubscriber } from '@verdant-web/common';
+
+/**
+ * A set of timers per-replica which are
+ * refreshed whenever the replica sends
+ * a message to the server. If a timer expires,
+ * the replica is removed from presence. This
+ * is a failsafe for socket-based replicas but
+ * essential for pull-based replicas.
+ */
+export class ReplicaKeepaliveTimers extends EventSubscriber<{
+	lost: (libraryId: string, replicaId: string) => void;
+}> {
+	private timers = new Map<string, NodeJS.Timeout>();
+
+	constructor(private timeout = 30 * 1000) {
+		super();
+	}
+
+	refresh = (libraryId: string, replicaId: string) => {
+		const existing = this.timers.get(replicaId);
+		if (existing) {
+			clearTimeout(existing);
+		}
+
+		this.timers.set(
+			replicaId,
+			setTimeout(() => {
+				this.emit('lost', libraryId, replicaId);
+				this.timers.delete(replicaId);
+			}, this.timeout),
+		);
+	};
+}

package/src/Replicas.ts

@@ -0,0 +1,238 @@
+import { Database } from 'better-sqlite3';
+import { TokenInfo } from './TokenVerifier.js';
+import { ReplicaInfoSpec } from './types.js';
+
+export class ReplicaInfos {
+	constructor(
+		private db: Database,
+		private readonly replicaTruancyMinutes: number,
+	) {}
+
+	private get truantCutoff() {
+		return Date.now() - this.replicaTruancyMinutes * 60 * 1000;
+	}
+
+	get = (libraryId: string, replicaId: string): ReplicaInfoSpec | null => {
+		const row = this.db
+			.prepare(
+				`
+					SELECT * FROM ReplicaInfo
+					WHERE id = ? AND libraryId = ?
+				`,
+			)
+			.get(replicaId, libraryId);
+		if (!row) {
+			return null;
+		}
+		return {
+			id: row.id,
+			libraryId: row.libraryId,
+			clientId: row.clientId,
+			lastSeenWallClockTime: row.lastSeenWallClockTime,
+			ackedLogicalTime: row.ackedLogicalTime,
+			type: row.type,
+			ackedServerOrder: row.ackedServerOrder,
+		};
+	};
+
+	getOrCreate = (
+		libraryId: string,
+		replicaId: string,
+		info: TokenInfo,
+	): {
+		status: 'new' | 'existing' | 'truant';
+		replicaInfo: ReplicaInfoSpec;
+	} => {
+		const replicaInfo: ReplicaInfoSpec = this.db
+			.prepare(
+				`
+			SELECT * FROM ReplicaInfo
+			WHERE id = ? AND libraryId = ?
+			`,
+			)
+			.get(replicaId, libraryId);
+
+		if (!replicaInfo) {
+			this.db
+				.prepare(
+					`
+			INSERT INTO ReplicaInfo (id, clientId, libraryId, type)
+			VALUES (?, ?, ?, ?)
+			`,
+				)
+				.run(replicaId, info.userId, libraryId, info.type);
+
+			return {
+				status: 'new',
+				replicaInfo: {
+					id: replicaId,
+					clientId: info.userId,
+					ackedLogicalTime: null,
+					lastSeenWallClockTime: null,
+					libraryId: libraryId,
+					type: info.type,
+					ackedServerOrder: 0,
+				},
+			};
+		}
+
+		if (replicaInfo.type !== info.type) {
+			// type should be updated if a new token changes it
+			this.db
+				.prepare(
+					`
+					UPDATE ReplicaInfo
+					SET type = ?
+					WHERE id = ? AND libraryId = ?
+					`,
+				)
+				.run(info.type, replicaId, libraryId);
+		}
+
+		if (replicaInfo.clientId !== info.userId) {
+			throw new Error(
+				`Replica ${replicaId} already exists with a different user ID`,
+			);
+		}
+
+		if (
+			replicaInfo.lastSeenWallClockTime &&
+			replicaInfo.lastSeenWallClockTime < this.truantCutoff
+		) {
+			return {
+				status: 'truant',
+				replicaInfo,
+			};
+		}
+
+		return {
+			status: 'existing',
+			replicaInfo,
+		};
+	};
+
+	getAllNonTruant = (libraryId: string): ReplicaInfoSpec[] => {
+		return this.db
+			.prepare(
+				`
+			SELECT * FROM ReplicaInfo
+			WHERE libraryId = ? AND lastSeenWallClockTime > ?
+		`,
+			)
+			.all(libraryId, this.truantCutoff);
+	};
+
+	updateLastSeen = (libraryId: string, replicaId: string) => {
+		const clockTime = new Date().getTime();
+		return this.db
+			.prepare(
+				`
+      UPDATE ReplicaInfo
+      SET lastSeenWallClockTime = ?
+      WHERE id = ? AND libraryId = ?
+    `,
+			)
+			.run(clockTime, replicaId, libraryId);
+	};
+
+	updateAcknowledged = (
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	) => {
+		this.db
+			.prepare(
+				`
+			UPDATE ReplicaInfo
+			SET ackedLogicalTime = ?
+			WHERE id = ? AND libraryId = ?
+		`,
+			)
+			.run(timestamp, replicaId, libraryId);
+	};
+
+	updateServerOrder = (
+		libraryId: string,
+		replicaId: string,
+		serverOrder: number,
+	) => {
+		// only increment the server order, never set it to lower than before
+		this.db
+			.prepare(
+				`
+			UPDATE ReplicaInfo
+			SET ackedServerOrder = MAX(ackedServerOrder, ?)
+			WHERE id = ? AND libraryId = ?
+		`,
+			)
+			.run(serverOrder, replicaId, libraryId);
+	};
+
+	acknowledgeOperation = (
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	) => {
+		if (!timestamp) return;
+
+		// get the server order from the operation with the timestamp and update the replica's server order
+		// to the max of the current server order and the server order of the operation
+		this.db
+			.prepare(
+				`
+			UPDATE ReplicaInfo
+			SET ackedLogicalTime = ?, ackedServerOrder = MAX(ackedServerOrder, COALESCE((
+				SELECT serverOrder FROM OperationHistory
+				WHERE libraryId = ? AND timestamp = ?
+			), 0))
+			WHERE id = ? AND libraryId = ?
+			`,
+			)
+			.run(timestamp, libraryId, timestamp, replicaId, libraryId);
+	};
+
+	getGlobalAck = (libraryId: string, onlineReplicaIds?: string[]) => {
+		const nonTruant = this.getAllNonTruant(libraryId);
+		const globalAckEligible = nonTruant.filter(
+			(replica) => replica.type < 2 || onlineReplicaIds?.includes(replica.id),
+		);
+		// get the earliest acked time
+		return globalAckEligible.reduce((acc, replica) => {
+			if (!replica.ackedLogicalTime) {
+				return acc;
+			}
+			if (acc === undefined) {
+				return replica.ackedLogicalTime;
+			}
+			return acc < replica.ackedLogicalTime ? acc : replica.ackedLogicalTime;
+		}, undefined as string | undefined);
+	};
+
+	delete = (libraryId: string, replicaId: string) => {
+		return this.db
+			.prepare(
+				`
+			DELETE FROM ReplicaInfo
+			WHERE id = ? AND libraryId = ?
+		`,
+			)
+			.run(replicaId, libraryId);
+	};
+
+	deleteAll = (libraryId: string) => {
+		return this.db
+			.prepare(
+				`
+			DELETE FROM ReplicaInfo
+			WHERE libraryId = ?
+		`,
+			)
+			.run(libraryId);
+	};
+
+	deleteAllForUser = (libraryId: string, userId: string) => {
+		return this.db
+			.prepare(`DELETE FROM ReplicaInfo WHERE libraryId = ? AND clientId = ?`)
+			.run(libraryId, userId);
+	};
+}