@verdant-web/server 3.0.5 → 3.1.1

package/src/storage/sqlShard/SqlOperations.ts

@@ -0,0 +1,146 @@
+import { Operation } from '@verdant-web/common';
+import { StoredOperation } from '../../types.js';
+import { OperationStorage } from '../Storage.js';
+import { Kysely } from 'kysely';
+import { Database, OperationHistoryRow } from './tables.js';
+import { Databases } from './Databases.js';
+
+export class SqlOperations implements OperationStorage {
+	constructor(
+		private dbs: Databases,
+		private dialect: 'postgres' | 'sqlite',
+	) {}
+
+	private hydrate = (row: OperationHistoryRow) => {
+		// avoiding extra allocation from .map by type-asserting here
+		row.data = JSON.parse(row.data);
+	};
+
+	getAll = async (
+		libraryId: string,
+		oid: string,
+	): Promise<StoredOperation[]> => {
+		const db = await this.dbs.get(libraryId);
+		const raw = await db
+			.selectFrom('OperationHistory')
+			.where('oid', '=', oid)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getBeforeServerOrder = async (
+		libraryId: string,
+		beforeServerOrder: number,
+	): Promise<StoredOperation[]> => {
+		const db = await this.dbs.get(libraryId);
+		const raw = await db
+			.selectFrom('OperationHistory')
+			.where('serverOrder', '<', beforeServerOrder)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getAfterServerOrder = async (
+		libraryId: string,
+		afterServerOrder: number,
+	): Promise<StoredOperation[]> => {
+		const db = await this.dbs.get(libraryId);
+		const raw = await db
+			.selectFrom('OperationHistory')
+			.where('serverOrder', '>', afterServerOrder)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getLatestServerOrder = async (libraryId: string): Promise<number> => {
+		const db = await this.dbs.get(libraryId);
+		return (
+			(
+				await db
+					.selectFrom('OperationHistory')
+					.orderBy('serverOrder', 'desc')
+					.select('serverOrder')
+					.limit(1)
+					.executeTakeFirst()
+			)?.serverOrder ?? 0
+		);
+	};
+
+	getCount = async (libraryId: string): Promise<number> => {
+		const db = await this.dbs.get(libraryId);
+		return (
+			(
+				await db
+					.selectFrom('OperationHistory')
+					.select(({ fn }) => fn.countAll<number>().as('count'))
+					.executeTakeFirst()
+			)?.count ?? 0
+		);
+	};
+
+	insertAll = async (
+		libraryId: string,
+		replicaId: string,
+		operations: Operation[],
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		// inserts all operations and updates server order
+		// FIXME: this whole thing is kinda sus
+		await db.transaction().execute(async (tx): Promise<void> => {
+			let orderResult = await tx
+				.selectFrom('OperationHistory')
+				.select(({ fn, val }) =>
+					fn.coalesce(fn.max<number>('serverOrder'), val(0)).as('serverOrder'),
+				)
+				.executeTakeFirst();
+			let currentServerOrder = orderResult?.serverOrder ?? 0;
+			for (const item of operations) {
+				await tx
+					.insertInto('OperationHistory')
+					.values({
+						oid: item.oid,
+						data: JSON.stringify(item.data),
+						timestamp: item.timestamp,
+						replicaId,
+						serverOrder: ++currentServerOrder,
+					})
+					.onConflict((cb) =>
+						cb.columns(['replicaId', 'oid', 'timestamp']).doNothing(),
+					)
+					.execute();
+			}
+		});
+	};
+
+	deleteAll = async (libraryId: string): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('OperationHistory').execute();
+	};
+	delete = async (
+		libraryId: string,
+		operations: Operation[],
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.transaction().execute(async (tx) => {
+			for (const item of operations) {
+				await tx
+					.deleteFrom('OperationHistory')
+					.where('oid', '=', item.oid)
+					.where('timestamp', '=', item.timestamp)
+					.execute();
+			}
+		});
+	};
+}

package/src/storage/sqlShard/SqlReplicas.ts

@@ -0,0 +1,237 @@
+import { StoredReplicaInfo } from '../../types.js';
+import { ReplicaStorage } from '../Storage.js';
+import { sql, type Kysely } from 'kysely';
+import { Database, ReplicaInfoRow } from './tables.js';
+import { ReplicaType, VerdantError } from '@verdant-web/common';
+import { Databases } from './Databases.js';
+
+export class SqlReplicas implements ReplicaStorage {
+	constructor(
+		private dbs: Databases,
+		private readonly replicaTruancyMinutes: number,
+		private readonly dialect: 'sqlite' | 'postgres',
+	) {}
+
+	get truantCutoff(): number {
+		return Date.now() - this.replicaTruancyMinutes * 60 * 1000;
+	}
+
+	private attachLibraryId = (libraryId: string, row: ReplicaInfoRow) => {
+		(row as any).libraryId = libraryId;
+		return row as StoredReplicaInfo;
+	};
+
+	get = async (
+		libraryId: string,
+		replicaId: string,
+	): Promise<StoredReplicaInfo | null> => {
+		const db = await this.dbs.get(libraryId);
+		const row =
+			(await db
+				.selectFrom('ReplicaInfo')
+				.where('id', '=', replicaId)
+				.selectAll()
+				.executeTakeFirst()) ?? null;
+		if (row) return this.attachLibraryId(libraryId, row);
+		return row;
+	};
+	getOrCreate = async (
+		libraryId: string,
+		replicaId: string,
+		info: { userId: string; type: ReplicaType },
+	): Promise<{
+		status: 'new' | 'existing' | 'truant';
+		replicaInfo: StoredReplicaInfo;
+	}> => {
+		const existing = await this.get(libraryId, replicaId);
+		if (!existing) {
+			const db = await this.dbs.get(libraryId);
+			const created = await db
+				.insertInto('ReplicaInfo')
+				.values({
+					id: replicaId,
+					clientId: info.userId,
+					type: info.type,
+					ackedServerOrder: 0,
+				})
+				.returningAll()
+				.executeTakeFirst();
+			if (!created) {
+				throw new VerdantError(
+					VerdantError.Code.Unexpected,
+					undefined,
+					'Failed to create replica',
+				);
+			}
+			return {
+				status: 'new',
+				replicaInfo: this.attachLibraryId(libraryId, created),
+			};
+		}
+
+		if (existing.type !== info.type) {
+			const db = await this.dbs.get(libraryId);
+			// type should be updated if a new token changes it
+			await db
+				.updateTable('ReplicaInfo')
+				.set({ type: info.type })
+				.where('id', '=', replicaId)
+				.execute();
+		}
+
+		if (existing.clientId !== info.userId) {
+			// replicas cannot change hands - this is a security issue
+			throw new VerdantError(
+				VerdantError.Code.Forbidden,
+				undefined,
+				'Another user is already using this replica ID',
+			);
+		}
+
+		if (
+			existing.lastSeenWallClockTime !== null &&
+			existing.lastSeenWallClockTime < this.truantCutoff
+		) {
+			return { status: 'truant', replicaInfo: existing };
+		}
+
+		return { status: 'existing', replicaInfo: existing };
+	};
+
+	getAll = async (
+		libraryId: string,
+		options?: { omitTruant: boolean } | undefined,
+	): Promise<StoredReplicaInfo[]> => {
+		const db = await this.dbs.get(libraryId);
+		let builder = db.selectFrom('ReplicaInfo');
+
+		if (options?.omitTruant) {
+			builder = builder.where('lastSeenWallClockTime', '>', this.truantCutoff);
+		}
+
+		return (await builder.selectAll().execute()).map(
+			this.attachLibraryId.bind(this, libraryId),
+		);
+	};
+
+	updateLastSeen = async (
+		libraryId: string,
+		replicaId: string,
+	): Promise<void> => {
+		const clockTime = Date.now();
+		const db = await this.dbs.get(libraryId);
+		await db
+			.updateTable('ReplicaInfo')
+			.set({ lastSeenWallClockTime: clockTime })
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	updateAckedServerOrder = async (
+		libraryId: string,
+		replicaId: string,
+		serverOrder: number,
+	): Promise<void> => {
+		const max = this.dialect === 'postgres' ? 'GREATEST' : 'MAX';
+		const db = await this.dbs.get(libraryId);
+		await db
+			.updateTable('ReplicaInfo')
+			.set(
+				'ackedServerOrder',
+				({ val }) =>
+					sql<number>`${sql.raw(max)}(ackedServerOrder, ${val(serverOrder)})`,
+			)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	updateAcknowledgedLogicalTime = async (
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db
+			.updateTable('ReplicaInfo')
+			.set({ ackedLogicalTime: timestamp })
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	getEarliestAckedServerOrder = async (libraryId: string): Promise<number> => {
+		const db = await this.dbs.get(libraryId);
+		// gets earliest acked server order of all non-truant replicas.
+		const res = await db
+			.selectFrom('ReplicaInfo')
+			.where('lastSeenWallClockTime', '>', this.truantCutoff)
+			.orderBy('ackedServerOrder', 'asc')
+			.select('ackedServerOrder')
+			.executeTakeFirst();
+		return res?.ackedServerOrder ?? 0;
+	};
+
+	acknowledgeOperation = async (
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	): Promise<void> => {
+		if (!timestamp) return;
+		const db = await this.dbs.get(libraryId);
+		// when acking an operation, we also set the replica's server order
+		// to that operation's server order, if it's greater.
+		const max = this.dialect === 'postgres' ? 'GREATEST' : 'MAX';
+		await db
+			.updateTable('ReplicaInfo')
+			.set({ ackedLogicalTime: timestamp })
+			.set(
+				'ackedServerOrder',
+				({ val }) => sql<number>`${sql.raw(max)}(
+					ackedServerOrder,
+					COALESCE(
+						(
+        			SELECT serverOrder FROM OperationHistory
+        			WHERE timestamp = ${val(timestamp)}
+      			),
+						0
+					)
+				)`,
+			)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	getGlobalAck = async (
+		libraryId: string,
+		onlineReplicaIds?: string[] | undefined,
+	): Promise<string | null> => {
+		const nonTruant = await this.getAll(libraryId, { omitTruant: true });
+		if (nonTruant.length === 0) return null;
+		const globalAckEligible = nonTruant.filter(
+			(replica) => replica.type < 2 || onlineReplicaIds?.includes(replica.id),
+		);
+
+		return globalAckEligible.reduce(
+			(acc, replica) => {
+				if (!replica.ackedLogicalTime) return acc;
+				if (acc === null) return replica.ackedLogicalTime;
+				return acc < replica.ackedLogicalTime ? acc : replica.ackedLogicalTime;
+			},
+			null as string | null,
+		);
+	};
+	delete = async (libraryId: string, replicaId: string): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('ReplicaInfo').where('id', '=', replicaId).execute();
+	};
+	deleteAll = async (libraryId: string): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('ReplicaInfo').execute();
+	};
+	deleteAllForUser = async (
+		libraryId: string,
+		userId: string,
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('ReplicaInfo').where('clientId', '=', userId).execute();
+	};
+}

package/src/storage/sqlShard/_testData/unifiedData.ts

@@ -0,0 +1,45 @@
+function random() {
+	return Math.random().toString(36).substring(7);
+}
+
+export function randomOperations(libraryId: string, count = 10) {
+	return Array.from({ length: count }, (_, i) => ({
+		oid: `oid-${i}-${random()}`,
+		timestamp: `timestamp-${i}-${random()}`,
+		data: `data-${i}-${random()}`,
+		serverOrder: i,
+		replicaId: `replicaId-${i}`,
+		libraryId,
+	}));
+}
+
+export function randomBaselines(libraryId: string, count = 10) {
+	return Array.from({ length: count }, (_, i) => ({
+		oid: `oid-${i}-${random()}`,
+		snapshot: `snapshot-${i}-${random()}`,
+		timestamp: `timestamp-${i}-${random()}`,
+		libraryId,
+	}));
+}
+
+export function randomFileMetadata(libraryId: string, count = 10) {
+	return Array.from({ length: count }, (_, i) => ({
+		libraryId,
+		fileId: `fileId-${i}-${random()}`,
+		name: `name-${i}-${random()}`,
+		type: `type-${i}`,
+		pendingDeleteAt: i,
+	}));
+}
+
+export function randomReplicaInfo(libraryId: string, count = 10) {
+	return Array.from({ length: count }, (_, i) => ({
+		libraryId,
+		id: `replicaId-${i}`,
+		clientId: `clientId-${i}-${random()}`,
+		lastSeenWallClockTime: i,
+		ackedLogicalTime: `ackedLogicalTime-${i}-${random()}`,
+		type: i,
+		ackedServerOrder: i,
+	}));
+}

package/src/storage/sqlShard/database.ts

@@ -0,0 +1,37 @@
+import { Kysely, SqliteDialect } from 'kysely';
+import { join } from 'path';
+import { Database as DatabaseTypes } from './tables.js';
+import Database from 'better-sqlite3';
+import { migrateToLatest } from '@a-type/kysely';
+import migrations from './migrations.js';
+
+export async function openDatabase(
+	directory: string,
+	libraryId: string,
+	options: {
+		skipMigrations?: boolean;
+		disableWal?: boolean;
+	} = {},
+) {
+	const label = `openDatabase ${libraryId}`;
+	console.time(label);
+	const filePath =
+		directory === ':memory:'
+			? ':memory:'
+			: join(directory, `${libraryId}.sqlite`);
+	const internalDb = new Database(filePath);
+	if (!options.disableWal) {
+		internalDb.pragma('journal_mode = WAL');
+	}
+	const db = new Kysely<DatabaseTypes>({
+		dialect: new SqliteDialect({
+			database: internalDb,
+		}),
+	});
+	// only migrate on first open
+	if (!options.skipMigrations) {
+		await migrateToLatest(db, migrations);
+	}
+	console.timeEnd(label);
+	return db;
+}

package/src/storage/sqlShard/migrations/v1.ts

@@ -0,0 +1,53 @@
+import { Kysely } from 'kysely';
+
+export async function up(db: Kysely<any>) {
+	await db.schema
+		.createTable('DocumentBaseline')
+		.ifNotExists()
+		.addColumn('oid', 'text', (cb) => cb.primaryKey())
+		.addColumn('snapshot', 'text')
+		.addColumn('timestamp', 'text', (cb) => cb.notNull())
+		.execute();
+
+	await db.schema
+		.createTable('OperationHistory')
+		.ifNotExists()
+		.addColumn('oid', 'text', (cb) => cb.notNull())
+		.addColumn('timestamp', 'text', (cb) => cb.notNull())
+		.addColumn('data', 'text', (cb) => cb.notNull())
+		.addColumn('serverOrder', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.addColumn('replicaId', 'text', (cb) => cb.notNull())
+		.addPrimaryKeyConstraint('OperationHistory_primaryKey', [
+			'replicaId',
+			'oid',
+			'timestamp',
+		])
+		.execute();
+
+	await db.schema
+		.createTable('ReplicaInfo')
+		.ifNotExists()
+		.addColumn('id', 'text', (cb) => cb.primaryKey())
+		.addColumn('clientId', 'text', (cb) => cb.notNull())
+		.addColumn('lastSeenWallClockTime', 'integer')
+		.addColumn('ackedLogicalTime', 'text')
+		.addColumn('type', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.addColumn('ackedServerOrder', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.execute();
+
+	await db.schema
+		.createTable('FileMetadata')
+		.ifNotExists()
+		.addColumn('fileId', 'text', (cb) => cb.primaryKey())
+		.addColumn('name', 'text', (cb) => cb.notNull())
+		.addColumn('type', 'text', (cb) => cb.notNull())
+		.addColumn('pendingDeleteAt', 'integer')
+		.execute();
+}
+
+export async function down(db: Kysely<any>) {
+	await db.schema.dropTable('DocumentBaseline').execute();
+	await db.schema.dropTable('OperationHistory').execute();
+	await db.schema.dropTable('ReplicaInfo').execute();
+	await db.schema.dropTable('FileMetadata').execute();
+}

package/src/storage/sqlShard/migrations.ts

@@ -0,0 +1,3 @@
+import * as v1 from './migrations/v1.js';
+
+export default { v1 };

package/src/storage/sqlShard/sqlShardStorage.ts

@@ -0,0 +1,81 @@
+import Database from 'better-sqlite3';
+import { Kysely, SqliteDialect } from 'kysely';
+import { StorageFactory } from '../Storage.js';
+import { SqlBaselines } from './SqlBaselines.js';
+import { SqlOperations } from './SqlOperations.js';
+import { SqlReplicas } from './SqlReplicas.js';
+import { Database as DatabaseTypes } from './tables.js';
+import { SqlFileMetadata } from './SqlFileMetadata.js';
+import { migrateToLatest } from '@a-type/kysely';
+import migrations from './migrations.js';
+import { Databases } from './Databases.js';
+import { transferToShards } from './transfer.js';
+import { existsSync, mkdirSync, readdirSync } from 'fs';
+
+export const sqlShardStorage = ({
+	databasesDirectory,
+	transferFromUnifiedDatabaseFile,
+	disableWal,
+	closeTimeout,
+}: {
+	databasesDirectory: string;
+	transferFromUnifiedDatabaseFile?: string;
+	disableWal?: boolean;
+	closeTimeout?: number;
+}): StorageFactory => {
+	let ready = Promise.resolve<void>(undefined);
+	if (!existsSync(databasesDirectory)) {
+		mkdirSync(databasesDirectory);
+		console.info(`Created databases directory: ${databasesDirectory}`);
+	}
+	if (transferFromUnifiedDatabaseFile) {
+		// check if directory is empty
+		const files =
+			databasesDirectory === ':memory:' ? [] : readdirSync(databasesDirectory);
+		if (files.length > 0) {
+			console.error(
+				`Cannot transfer from unified database to non-empty directory: ${databasesDirectory}. This might mean the transfer has already happened and you're free to turn off the transferFromUnifiedDatabaseFile option.`,
+			);
+		} else {
+			ready = transferToShards({
+				file: transferFromUnifiedDatabaseFile,
+				directory: databasesDirectory,
+			}).then();
+		}
+	}
+	const dbs = new Databases({
+		directory: databasesDirectory,
+		disableWal,
+		closeTimeout,
+	});
+	return (options) => {
+		const baselines = new SqlBaselines(dbs, 'sqlite');
+		const operations = new SqlOperations(dbs, 'sqlite');
+		const replicas = new SqlReplicas(
+			dbs,
+			options.replicaTruancyMinutes,
+			'sqlite',
+		);
+		const fileMetadata = new SqlFileMetadata(
+			dbs,
+			options.fileDeleteExpirationDays,
+			'sqlite',
+		);
+		const close = async () => {
+			storage.open = false;
+			await dbs.destroy();
+		};
+		const storage = {
+			baselines,
+			operations,
+			replicas,
+			fileMetadata,
+			close,
+			open: false,
+			ready: ready.then(() => {
+				storage.open = true;
+			}),
+		};
+		return storage;
+	};
+};

package/src/storage/sqlShard/tables.ts

@@ -0,0 +1,45 @@
+import { ReplicaType } from '@verdant-web/common';
+import { Generated, Selectable } from 'kysely';
+
+export interface Database {
+	OperationHistory: OperationHistoryTable;
+	DocumentBaseline: DocumentBaselineTable;
+	ReplicaInfo: ReplicaInfoTable;
+	FileMetadata: FileMetadataTable;
+}
+
+export interface OperationHistoryTable {
+	oid: string;
+	timestamp: string;
+	data: string;
+	serverOrder: number;
+	replicaId: string;
+}
+export type OperationHistoryRow = Selectable<OperationHistoryTable>;
+
+export interface DocumentBaselineTable {
+	oid: string;
+	snapshot: string;
+	timestamp: string;
+}
+export type DocumentBaselineRow = Selectable<DocumentBaselineTable>;
+
+export interface ReplicaInfoTable {
+	id: string;
+	clientId: string;
+	lastSeenWallClockTime: number | null;
+	ackedLogicalTime: string | null;
+	type: ReplicaType;
+	ackedServerOrder: Generated<number>;
+}
+
+export type ReplicaInfoRow = Selectable<ReplicaInfoTable>;
+
+export interface FileMetadataTable {
+	fileId: string;
+	name: string;
+	type: string;
+	pendingDeleteAt: number | null;
+}
+
+export type FileMetadataRow = Selectable<FileMetadataTable>;