@verdant-web/server 2.1.3 → 3.0.0

package/src/storage/Storage.ts

@@ -0,0 +1,118 @@
+import {
+	DocumentBaseline,
+	Operation,
+	Ref,
+	ReplicaType,
+} from '@verdant-web/common';
+import {
+	HydratedDocumentBaseline,
+	StoredOperation,
+	StoredReplicaInfo,
+	FileMetadata,
+} from '../types.js';
+import { FileInfo } from '../files/FileStorage.js';
+
+export interface ReplicaStorage {
+	readonly truantCutoff: number;
+	get(libraryId: string, replicaId: string): Promise<StoredReplicaInfo | null>;
+	getOrCreate(
+		libraryId: string,
+		replicaId: string,
+		info: { userId: string; type: ReplicaType },
+	): Promise<{
+		status: 'new' | 'existing' | 'truant';
+		replicaInfo: StoredReplicaInfo;
+	}>;
+	getAll(
+		libraryId: string,
+		options?: { omitTruant: boolean },
+	): Promise<StoredReplicaInfo[]>;
+	updateLastSeen(libraryId: string, replicaId: string): Promise<void>;
+	updateAckedServerOrder(
+		libraryId: string,
+		replicaId: string,
+		serverOrder: number,
+	): Promise<void>;
+	updateAcknowledgedLogicalTime(
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	): Promise<void>;
+	getEarliestAckedServerOrder(libraryId: string): Promise<number>;
+	acknowledgeOperation(
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	): Promise<void>;
+	getGlobalAck(
+		libraryId: string,
+		onlineReplicaIds?: string[],
+	): Promise<string | null>;
+	delete(libraryId: string, replicaId: string): Promise<void>;
+	deleteAll(libraryId: string): Promise<void>;
+	deleteAllForUser(libraryId: string, userId: string): Promise<void>;
+}
+
+export interface OperationStorage {
+	getAll(libraryId: string, oid: string): Promise<StoredOperation[]>;
+	getBeforeServerOrder(
+		libraryId: string,
+		beforeServerOrder: number,
+	): Promise<StoredOperation[]>;
+	getAfterServerOrder(
+		libraryId: string,
+		afterServerOrder: number,
+	): Promise<StoredOperation[]>;
+	getLatestServerOrder(libraryId: string): Promise<number>;
+	getCount(libraryId: string): Promise<number>;
+	insertAll(
+		libraryId: string,
+		replicaId: string,
+		operations: Operation[],
+	): Promise<void>;
+	deleteAll(libraryId: string): Promise<void>;
+	delete(libraryId: string, operations: Operation[]): Promise<void>;
+}
+
+export interface BaselineStorage {
+	get(libraryId: string, oid: string): Promise<HydratedDocumentBaseline | null>;
+	getAll(libraryId: string): Promise<HydratedDocumentBaseline[]>;
+	insertAll(
+		libraryId: string,
+		baselines: DocumentBaseline<any>[],
+	): Promise<void>;
+	deleteAll(libraryId: string): Promise<void>;
+	getCount(libraryId: string): Promise<number>;
+	applyOperations(
+		libraryId: string,
+		oid: string,
+		operations: StoredOperation[],
+		deletedRefs?: Ref[],
+	): Promise<void>;
+}
+
+export interface FileMetadataStorage {
+	get(libraryId: string, fileId: string): Promise<FileMetadata | null>;
+	getAll(libraryId: string): Promise<FileMetadata[]>;
+	deleteAll(libraryId: string): Promise<void>;
+	put(libraryId: string, fileInfo: FileInfo): Promise<void>;
+	markPendingDelete(libraryId: string, fileId: string): Promise<void>;
+	delete(libraryId: string, fileId: string): Promise<void>;
+	getPendingDelete(libraryId: string): Promise<FileMetadata[]>;
+}
+
+export interface StorageOptions {
+	replicaTruancyMinutes: number;
+	fileDeleteExpirationDays: number;
+}
+export interface Storage {
+	replicas: ReplicaStorage;
+	operations: OperationStorage;
+	baselines: BaselineStorage;
+	fileMetadata: FileMetadataStorage;
+	close(): Promise<void>;
+	readonly open: boolean;
+	readonly ready: Promise<void>;
+}
+
+export type StorageFactory = (options: StorageOptions) => Storage;

package/src/storage/index.ts

@@ -0,0 +1,2 @@
+export * from './Storage.js';
+export * from './sql/sqlStorage.js';

package/src/storage/sql/SqlBaselines.ts

@@ -0,0 +1,141 @@
+import { Kysely } from 'kysely';
+import { BaselineStorage } from '../Storage.js';
+import { Database, DocumentBaselineRow } from './tables.js';
+import { DocumentBaseline, Ref, applyPatch } from '@verdant-web/common';
+import { HydratedDocumentBaseline, StoredOperation } from '../../types.js';
+
+export class SqlBaselines implements BaselineStorage {
+	constructor(
+		private db: Kysely<Database>,
+		private dialect: 'postgres' | 'sqlite',
+	) {}
+
+	get = async (
+		libraryId: string,
+		oid: string,
+		{ tx }: { tx?: Kysely<Database> } = {},
+	): Promise<HydratedDocumentBaseline | null> => {
+		const db = tx ?? this.db;
+		const raw =
+			(await db
+				.selectFrom('DocumentBaseline')
+				.where('libraryId', '=', libraryId)
+				.where('oid', '=', oid)
+				.selectAll()
+				.executeTakeFirst()) ?? null;
+		if (!raw) return null;
+		this.hydrate(raw);
+		return raw as any;
+	};
+
+	getAll = async (libraryId: string) => {
+		const raw = await this.db
+			.selectFrom('DocumentBaseline')
+			.where('libraryId', '=', libraryId)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+
+		return raw as any;
+	};
+
+	set = async (
+		libraryId: string,
+		baseline: DocumentBaseline,
+		{
+			tx,
+		}: {
+			tx?: Kysely<Database>;
+		},
+	) => {
+		const db = tx ?? this.db;
+		if (!baseline.snapshot) {
+			await db
+				.deleteFrom('DocumentBaseline')
+				.where('libraryId', '=', libraryId)
+				.where('oid', '=', baseline.oid)
+				.execute();
+			return;
+		}
+
+		await db
+			.insertInto('DocumentBaseline')
+			.values({
+				libraryId,
+				oid: baseline.oid,
+				snapshot: JSON.stringify(baseline.snapshot),
+				timestamp: baseline.timestamp,
+			})
+			.onConflict((cb) =>
+				cb.doUpdateSet({
+					snapshot: JSON.stringify(baseline.snapshot),
+					timestamp: baseline.timestamp,
+				}),
+			)
+			.execute();
+	};
+
+	insertAll = async (libraryId: string, baselines: DocumentBaseline[]) => {
+		await this.db.transaction().execute(async (tx) => {
+			for (const baseline of baselines) {
+				await this.set(libraryId, baseline, { tx });
+			}
+		});
+	};
+
+	private hydrate = (row: DocumentBaselineRow) => {
+		// avoiding extra allocation from .map by type-asserting here
+		row.snapshot = JSON.parse(row.snapshot);
+	};
+
+	applyOperations = async (
+		libraryId: string,
+		oid: string,
+		operations: StoredOperation[],
+		deletedRefs?: Ref[],
+	) => {
+		if (operations.length === 0) return;
+
+		await this.db.transaction().execute(async (tx) => {
+			let baseline = await this.get(libraryId, oid, { tx });
+			if (!baseline) {
+				baseline = {
+					oid,
+					snapshot: {},
+					timestamp: operations[0].timestamp,
+					libraryId,
+				};
+			}
+			for (const operation of operations) {
+				baseline.snapshot = applyPatch(
+					baseline.snapshot,
+					operation.data,
+					deletedRefs,
+				);
+			}
+			baseline.timestamp = operations[operations.length - 1].timestamp;
+			await this.set(libraryId, baseline, { tx });
+		});
+	};
+
+	deleteAll = async (libraryId: string) => {
+		await this.db
+			.deleteFrom('DocumentBaseline')
+			.where('libraryId', '=', libraryId)
+			.execute();
+	};
+
+	getCount = async (libraryId: string) => {
+		return (
+			(
+				await this.db
+					.selectFrom('DocumentBaseline')
+					.where('libraryId', '=', libraryId)
+					.select(({ fn }) => fn.countAll<number>().as('count'))
+					.executeTakeFirst()
+			)?.count ?? 0
+		);
+	};
+}

package/src/storage/sql/SqlFileMetadata.ts

@@ -0,0 +1,94 @@
+import { Kysely } from 'kysely';
+import { FileMetadataStorage } from '../Storage.js';
+import { Database } from './tables.js';
+import { FileMetadata } from '../../types.js';
+import { FileInfo } from '../../files/FileStorage.js';
+
+export class SqlFileMetadata implements FileMetadataStorage {
+	constructor(
+		private db: Kysely<Database>,
+		private deleteExpirationDays: number,
+		private dialect: 'postgres' | 'sqlite',
+	) {}
+
+	get = async (
+		libraryId: string,
+		fileId: string,
+	): Promise<FileMetadata | null> => {
+		return (
+			(await this.db
+				.selectFrom('FileMetadata')
+				.where('libraryId', '=', libraryId)
+				.where('fileId', '=', fileId)
+				.selectAll()
+				.executeTakeFirst()) ?? null
+		);
+	};
+
+	getAll = async (libraryId: string): Promise<FileMetadata[]> => {
+		return this.db
+			.selectFrom('FileMetadata')
+			.where('libraryId', '=', libraryId)
+			.selectAll()
+			.execute();
+	};
+
+	deleteAll = async (libraryId: string): Promise<void> => {
+		await this.db
+			.deleteFrom('FileMetadata')
+			.where('libraryId', '=', libraryId)
+			.execute();
+	};
+
+	put = async (libraryId: string, fileInfo: FileInfo): Promise<void> => {
+		await this.db
+			.insertInto('FileMetadata')
+			.values({
+				libraryId,
+				fileId: fileInfo.id,
+				name: fileInfo.fileName,
+				type: fileInfo.type,
+			})
+			.onConflict((cb) =>
+				cb.columns(['libraryId', 'fileId']).doUpdateSet({
+					name: fileInfo.fileName,
+					type: fileInfo.type,
+					pendingDeleteAt: null,
+				}),
+			)
+			.execute();
+	};
+
+	markPendingDelete = async (
+		libraryId: string,
+		fileId: string,
+	): Promise<void> => {
+		await this.db
+			.updateTable('FileMetadata')
+			.set({ pendingDeleteAt: Date.now() })
+			.where('libraryId', '=', libraryId)
+			.where('fileId', '=', fileId)
+			.execute();
+	};
+
+	delete = async (libraryId: string, fileId: string): Promise<void> => {
+		await this.db
+			.deleteFrom('FileMetadata')
+			.where('libraryId', '=', libraryId)
+			.where('fileId', '=', fileId)
+			.execute();
+	};
+
+	getPendingDelete = async (libraryId: string): Promise<FileMetadata[]> => {
+		return this.db
+			.selectFrom('FileMetadata')
+			.where('libraryId', '=', libraryId)
+			.where(
+				'pendingDeleteAt',
+				'<',
+				Date.now() - 1000 * 60 * 60 * 24 * this.deleteExpirationDays,
+			)
+			.selectAll()
+			.execute();
+	};
+}

package/src/storage/sql/SqlOperations.ts

@@ -0,0 +1,149 @@
+import { Operation } from '@verdant-web/common';
+import { StoredOperation } from '../../types.js';
+import { OperationStorage } from '../Storage.js';
+import { Kysely } from 'kysely';
+import { Database, OperationHistoryRow } from './tables.js';
+
+export class SqlOperations implements OperationStorage {
+	constructor(
+		private db: Kysely<Database>,
+		private dialect: 'postgres' | 'sqlite',
+	) {}
+
+	private hydrate = (row: OperationHistoryRow) => {
+		// avoiding extra allocation from .map by type-asserting here
+		row.data = JSON.parse(row.data);
+	};
+
+	getAll = async (
+		libraryId: string,
+		oid: string,
+	): Promise<StoredOperation[]> => {
+		const raw = await this.db
+			.selectFrom('OperationHistory')
+			.where('libraryId', '=', libraryId)
+			.where('oid', '=', oid)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getBeforeServerOrder = async (
+		libraryId: string,
+		beforeServerOrder: number,
+	): Promise<StoredOperation[]> => {
+		const raw = await this.db
+			.selectFrom('OperationHistory')
+			.where('libraryId', '=', libraryId)
+			.where('serverOrder', '<', beforeServerOrder)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getAfterServerOrder = async (
+		libraryId: string,
+		afterServerOrder: number,
+	): Promise<StoredOperation[]> => {
+		const raw = await this.db
+			.selectFrom('OperationHistory')
+			.where('libraryId', '=', libraryId)
+			.where('serverOrder', '>', afterServerOrder)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getLatestServerOrder = async (libraryId: string): Promise<number> => {
+		return (
+			(
+				await this.db
+					.selectFrom('OperationHistory')
+					.where('libraryId', '=', libraryId)
+					.orderBy('serverOrder', 'desc')
+					.select('serverOrder')
+					.limit(1)
+					.executeTakeFirst()
+			)?.serverOrder ?? 0
+		);
+	};
+
+	getCount = async (libraryId: string): Promise<number> => {
+		return (
+			(
+				await this.db
+					.selectFrom('OperationHistory')
+					.where('libraryId', '=', libraryId)
+					.select(({ fn }) => fn.countAll<number>().as('count'))
+					.executeTakeFirst()
+			)?.count ?? 0
+		);
+	};
+
+	insertAll = async (
+		libraryId: string,
+		replicaId: string,
+		operations: Operation[],
+	): Promise<void> => {
+		// inserts all operations and updates server order
+		// FIXME: this whole thing is kinda sus
+		await this.db.transaction().execute(async (tx): Promise<void> => {
+			let orderResult = await tx
+				.selectFrom('OperationHistory')
+				.select(({ fn, val }) =>
+					fn.coalesce(fn.max<number>('serverOrder'), val(0)).as('serverOrder'),
+				)
+				.where('libraryId', '=', libraryId)
+				.executeTakeFirst();
+			let currentServerOrder = orderResult?.serverOrder ?? 0;
+			for (const item of operations) {
+				await tx
+					.insertInto('OperationHistory')
+					.values({
+						libraryId,
+						oid: item.oid,
+						data: JSON.stringify(item.data),
+						timestamp: item.timestamp,
+						replicaId,
+						serverOrder: ++currentServerOrder,
+					})
+					.onConflict((cb) =>
+						cb
+							.columns(['libraryId', 'replicaId', 'oid', 'timestamp'])
+							.doNothing(),
+					)
+					.execute();
+			}
+		});
+	};
+
+	deleteAll = async (libraryId: string): Promise<void> => {
+		await this.db
+			.deleteFrom('OperationHistory')
+			.where('libraryId', '=', libraryId)
+			.execute();
+	};
+	delete = async (
+		libraryId: string,
+		operations: Operation[],
+	): Promise<void> => {
+		await this.db
+			.deleteFrom('OperationHistory')
+			.where('libraryId', '=', libraryId)
+			.where(
+				'oid',
+				'in',
+				operations.map((o) => o.oid),
+			)
+			.execute();
+	};
+}