@verdant-web/server 3.0.4 → 3.1.0

package/src/storage/sqlShard/SqlReplicas.ts

@@ -0,0 +1,237 @@
+import { StoredReplicaInfo } from '../../types.js';
+import { ReplicaStorage } from '../Storage.js';
+import { sql, type Kysely } from 'kysely';
+import { Database, ReplicaInfoRow } from './tables.js';
+import { ReplicaType, VerdantError } from '@verdant-web/common';
+import { Databases } from './Databases.js';
+
+export class SqlReplicas implements ReplicaStorage {
+	constructor(
+		private dbs: Databases,
+		private readonly replicaTruancyMinutes: number,
+		private readonly dialect: 'sqlite' | 'postgres',
+	) {}
+
+	get truantCutoff(): number {
+		return Date.now() - this.replicaTruancyMinutes * 60 * 1000;
+	}
+
+	private attachLibraryId = (libraryId: string, row: ReplicaInfoRow) => {
+		(row as any).libraryId = libraryId;
+		return row as StoredReplicaInfo;
+	};
+
+	get = async (
+		libraryId: string,
+		replicaId: string,
+	): Promise<StoredReplicaInfo | null> => {
+		const db = await this.dbs.get(libraryId);
+		const row =
+			(await db
+				.selectFrom('ReplicaInfo')
+				.where('id', '=', replicaId)
+				.selectAll()
+				.executeTakeFirst()) ?? null;
+		if (row) return this.attachLibraryId(libraryId, row);
+		return row;
+	};
+	getOrCreate = async (
+		libraryId: string,
+		replicaId: string,
+		info: { userId: string; type: ReplicaType },
+	): Promise<{
+		status: 'new' | 'existing' | 'truant';
+		replicaInfo: StoredReplicaInfo;
+	}> => {
+		const existing = await this.get(libraryId, replicaId);
+		if (!existing) {
+			const db = await this.dbs.get(libraryId);
+			const created = await db
+				.insertInto('ReplicaInfo')
+				.values({
+					id: replicaId,
+					clientId: info.userId,
+					type: info.type,
+					ackedServerOrder: 0,
+				})
+				.returningAll()
+				.executeTakeFirst();
+			if (!created) {
+				throw new VerdantError(
+					VerdantError.Code.Unexpected,
+					undefined,
+					'Failed to create replica',
+				);
+			}
+			return {
+				status: 'new',
+				replicaInfo: this.attachLibraryId(libraryId, created),
+			};
+		}
+
+		if (existing.type !== info.type) {
+			const db = await this.dbs.get(libraryId);
+			// type should be updated if a new token changes it
+			await db
+				.updateTable('ReplicaInfo')
+				.set({ type: info.type })
+				.where('id', '=', replicaId)
+				.execute();
+		}
+
+		if (existing.clientId !== info.userId) {
+			// replicas cannot change hands - this is a security issue
+			throw new VerdantError(
+				VerdantError.Code.Forbidden,
+				undefined,
+				'Another user is already using this replica ID',
+			);
+		}
+
+		if (
+			existing.lastSeenWallClockTime !== null &&
+			existing.lastSeenWallClockTime < this.truantCutoff
+		) {
+			return { status: 'truant', replicaInfo: existing };
+		}
+
+		return { status: 'existing', replicaInfo: existing };
+	};
+
+	getAll = async (
+		libraryId: string,
+		options?: { omitTruant: boolean } | undefined,
+	): Promise<StoredReplicaInfo[]> => {
+		const db = await this.dbs.get(libraryId);
+		let builder = db.selectFrom('ReplicaInfo');
+
+		if (options?.omitTruant) {
+			builder = builder.where('lastSeenWallClockTime', '>', this.truantCutoff);
+		}
+
+		return (await builder.selectAll().execute()).map(
+			this.attachLibraryId.bind(this, libraryId),
+		);
+	};
+
+	updateLastSeen = async (
+		libraryId: string,
+		replicaId: string,
+	): Promise<void> => {
+		const clockTime = Date.now();
+		const db = await this.dbs.get(libraryId);
+		await db
+			.updateTable('ReplicaInfo')
+			.set({ lastSeenWallClockTime: clockTime })
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	updateAckedServerOrder = async (
+		libraryId: string,
+		replicaId: string,
+		serverOrder: number,
+	): Promise<void> => {
+		const max = this.dialect === 'postgres' ? 'GREATEST' : 'MAX';
+		const db = await this.dbs.get(libraryId);
+		await db
+			.updateTable('ReplicaInfo')
+			.set(
+				'ackedServerOrder',
+				({ val }) =>
+					sql<number>`${sql.raw(max)}(ackedServerOrder, ${val(serverOrder)})`,
+			)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	updateAcknowledgedLogicalTime = async (
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db
+			.updateTable('ReplicaInfo')
+			.set({ ackedLogicalTime: timestamp })
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	getEarliestAckedServerOrder = async (libraryId: string): Promise<number> => {
+		const db = await this.dbs.get(libraryId);
+		// gets earliest acked server order of all non-truant replicas.
+		const res = await db
+			.selectFrom('ReplicaInfo')
+			.where('lastSeenWallClockTime', '>', this.truantCutoff)
+			.orderBy('ackedServerOrder', 'asc')
+			.select('ackedServerOrder')
+			.executeTakeFirst();
+		return res?.ackedServerOrder ?? 0;
+	};
+
+	acknowledgeOperation = async (
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	): Promise<void> => {
+		if (!timestamp) return;
+		const db = await this.dbs.get(libraryId);
+		// when acking an operation, we also set the replica's server order
+		// to that operation's server order, if it's greater.
+		const max = this.dialect === 'postgres' ? 'GREATEST' : 'MAX';
+		await db
+			.updateTable('ReplicaInfo')
+			.set({ ackedLogicalTime: timestamp })
+			.set(
+				'ackedServerOrder',
+				({ val }) => sql<number>`${sql.raw(max)}(
+					ackedServerOrder,
+					COALESCE(
+						(
+        			SELECT serverOrder FROM OperationHistory
+        			WHERE timestamp = ${val(timestamp)}
+      			),
+						0
+					)
+				)`,
+			)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	getGlobalAck = async (
+		libraryId: string,
+		onlineReplicaIds?: string[] | undefined,
+	): Promise<string | null> => {
+		const nonTruant = await this.getAll(libraryId, { omitTruant: true });
+		if (nonTruant.length === 0) return null;
+		const globalAckEligible = nonTruant.filter(
+			(replica) => replica.type < 2 || onlineReplicaIds?.includes(replica.id),
+		);
+
+		return globalAckEligible.reduce(
+			(acc, replica) => {
+				if (!replica.ackedLogicalTime) return acc;
+				if (acc === null) return replica.ackedLogicalTime;
+				return acc < replica.ackedLogicalTime ? acc : replica.ackedLogicalTime;
+			},
+			null as string | null,
+		);
+	};
+	delete = async (libraryId: string, replicaId: string): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('ReplicaInfo').where('id', '=', replicaId).execute();
+	};
+	deleteAll = async (libraryId: string): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('ReplicaInfo').execute();
+	};
+	deleteAllForUser = async (
+		libraryId: string,
+		userId: string,
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('ReplicaInfo').where('clientId', '=', userId).execute();
+	};
+}

package/src/storage/sqlShard/_testData/unifiedData.ts

@@ -0,0 +1,52 @@
+import {
+	OperationHistoryRow,
+	DocumentBaselineRow,
+	FileMetadataRow,
+	ReplicaInfoRow,
+} from '../../sql/tables.js';
+
+function random() {
+	return Math.random().toString(36).substring(7);
+}
+
+export function randomOperations(libraryId: string) {
+	return Array.from({ length: 10 }, (_, i) => ({
+		oid: `oid-${i}-${random()}`,
+		timestamp: `timestamp-${i}-${random()}`,
+		data: `data-${i}-${random()}`,
+		serverOrder: i,
+		replicaId: `replicaId-${i}`,
+		libraryId,
+	}));
+}
+
+export function randomBaselines(libraryId: string) {
+	return Array.from({ length: 10 }, (_, i) => ({
+		oid: `oid-${i}-${random()}`,
+		snapshot: `snapshot-${i}-${random()}`,
+		timestamp: `timestamp-${i}-${random()}`,
+		libraryId,
+	}));
+}
+
+export function randomFileMetadata(libraryId: string) {
+	return Array.from({ length: 10 }, (_, i) => ({
+		libraryId,
+		fileId: `fileId-${i}-${random()}`,
+		name: `name-${i}-${random()}`,
+		type: `type-${i}`,
+		pendingDeleteAt: i,
+	}));
+}
+
+export function randomReplicaInfo(libraryId: string) {
+	return Array.from({ length: 10 }, (_, i) => ({
+		libraryId,
+		id: `replicaId-${i}`,
+		clientId: `clientId-${i}-${random()}`,
+		lastSeenWallClockTime: i,
+		ackedLogicalTime: `ackedLogicalTime-${i}-${random()}`,
+		type: i,
+		ackedServerOrder: i,
+	}));
+}

package/src/storage/sqlShard/database.ts

@@ -0,0 +1,37 @@
+import { Kysely, SqliteDialect } from 'kysely';
+import { join } from 'path';
+import { Database as DatabaseTypes } from './tables.js';
+import Database from 'better-sqlite3';
+import { migrateToLatest } from '@a-type/kysely';
+import migrations from './migrations.js';
+
+export async function openDatabase(
+	directory: string,
+	libraryId: string,
+	options: {
+		skipMigrations?: boolean;
+		disableWal?: boolean;
+	} = {},
+) {
+	const label = `openDatabase ${libraryId}`;
+	console.time(label);
+	const filePath =
+		directory === ':memory:'
+			? ':memory:'
+			: join(directory, `${libraryId}.sqlite`);
+	const internalDb = new Database(filePath);
+	if (!options.disableWal) {
+		internalDb.pragma('journal_mode = WAL');
+	}
+	const db = new Kysely<DatabaseTypes>({
+		dialect: new SqliteDialect({
+			database: internalDb,
+		}),
+	});
+	// only migrate on first open
+	if (!options.skipMigrations) {
+		await migrateToLatest(db, migrations);
+	}
+	console.timeEnd(label);
+	return db;
+}

package/src/storage/sqlShard/migrations/v1.ts

@@ -0,0 +1,53 @@
+import { Kysely } from 'kysely';
+
+export async function up(db: Kysely<any>) {
+	await db.schema
+		.createTable('DocumentBaseline')
+		.ifNotExists()
+		.addColumn('oid', 'text', (cb) => cb.primaryKey())
+		.addColumn('snapshot', 'text')
+		.addColumn('timestamp', 'text', (cb) => cb.notNull())
+		.execute();
+
+	await db.schema
+		.createTable('OperationHistory')
+		.ifNotExists()
+		.addColumn('oid', 'text', (cb) => cb.notNull())
+		.addColumn('timestamp', 'text', (cb) => cb.notNull())
+		.addColumn('data', 'text', (cb) => cb.notNull())
+		.addColumn('serverOrder', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.addColumn('replicaId', 'text', (cb) => cb.notNull())
+		.addPrimaryKeyConstraint('OperationHistory_primaryKey', [
+			'replicaId',
+			'oid',
+			'timestamp',
+		])
+		.execute();
+
+	await db.schema
+		.createTable('ReplicaInfo')
+		.ifNotExists()
+		.addColumn('id', 'text', (cb) => cb.primaryKey())
+		.addColumn('clientId', 'text', (cb) => cb.notNull())
+		.addColumn('lastSeenWallClockTime', 'integer')
+		.addColumn('ackedLogicalTime', 'text')
+		.addColumn('type', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.addColumn('ackedServerOrder', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.execute();
+
+	await db.schema
+		.createTable('FileMetadata')
+		.ifNotExists()
+		.addColumn('fileId', 'text', (cb) => cb.primaryKey())
+		.addColumn('name', 'text', (cb) => cb.notNull())
+		.addColumn('type', 'text', (cb) => cb.notNull())
+		.addColumn('pendingDeleteAt', 'integer')
+		.execute();
+}
+
+export async function down(db: Kysely<any>) {
+	await db.schema.dropTable('DocumentBaseline').execute();
+	await db.schema.dropTable('OperationHistory').execute();
+	await db.schema.dropTable('ReplicaInfo').execute();
+	await db.schema.dropTable('FileMetadata').execute();
+}

package/src/storage/sqlShard/migrations.ts

@@ -0,0 +1,3 @@
+import * as v1 from './migrations/v1.js';
+
+export default { v1 };

package/src/storage/sqlShard/sqlShardStorage.ts

@@ -0,0 +1,81 @@
+import Database from 'better-sqlite3';
+import { Kysely, SqliteDialect } from 'kysely';
+import { StorageFactory } from '../Storage.js';
+import { SqlBaselines } from './SqlBaselines.js';
+import { SqlOperations } from './SqlOperations.js';
+import { SqlReplicas } from './SqlReplicas.js';
+import { Database as DatabaseTypes } from './tables.js';
+import { SqlFileMetadata } from './SqlFileMetadata.js';
+import { migrateToLatest } from '@a-type/kysely';
+import migrations from './migrations.js';
+import { Databases } from './Databases.js';
+import { transferToShards } from './transfer.js';
+import { existsSync, mkdirSync, readdirSync } from 'fs';
+
+export const sqlShardStorage = ({
+	databasesDirectory,
+	transferFromUnifiedDatabaseFile,
+	disableWal,
+	closeTimeout,
+}: {
+	databasesDirectory: string;
+	transferFromUnifiedDatabaseFile?: string;
+	disableWal?: boolean;
+	closeTimeout?: number;
+}): StorageFactory => {
+	let ready = Promise.resolve<void>(undefined);
+	if (!existsSync(databasesDirectory)) {
+		mkdirSync(databasesDirectory);
+		console.info(`Created databases directory: ${databasesDirectory}`);
+	}
+	if (transferFromUnifiedDatabaseFile) {
+		// check if directory is empty
+		const files =
+			databasesDirectory === ':memory:' ? [] : readdirSync(databasesDirectory);
+		if (files.length > 0) {
+			console.error(
+				`Cannot transfer from unified database to non-empty directory: ${databasesDirectory}. This might mean the transfer has already happened and you're free to turn off the transferFromUnifiedDatabaseFile option.`,
+			);
+		} else {
+			ready = transferToShards({
+				file: transferFromUnifiedDatabaseFile,
+				directory: databasesDirectory,
+			}).then();
+		}
+	}
+	const dbs = new Databases({
+		directory: databasesDirectory,
+		disableWal,
+		closeTimeout,
+	});
+	return (options) => {
+		const baselines = new SqlBaselines(dbs, 'sqlite');
+		const operations = new SqlOperations(dbs, 'sqlite');
+		const replicas = new SqlReplicas(
+			dbs,
+			options.replicaTruancyMinutes,
+			'sqlite',
+		);
+		const fileMetadata = new SqlFileMetadata(
+			dbs,
+			options.fileDeleteExpirationDays,
+			'sqlite',
+		);
+		const close = async () => {
+			storage.open = false;
+			await dbs.destroy();
+		};
+		const storage = {
+			baselines,
+			operations,
+			replicas,
+			fileMetadata,
+			close,
+			open: false,
+			ready: ready.then(() => {
+				storage.open = true;
+			}),
+		};
+		return storage;
+	};
+};

package/src/storage/sqlShard/tables.ts

@@ -0,0 +1,45 @@
+import { ReplicaType } from '@verdant-web/common';
+import { Generated, Selectable } from 'kysely';
+
+export interface Database {
+	OperationHistory: OperationHistoryTable;
+	DocumentBaseline: DocumentBaselineTable;
+	ReplicaInfo: ReplicaInfoTable;
+	FileMetadata: FileMetadataTable;
+}
+
+export interface OperationHistoryTable {
+	oid: string;
+	timestamp: string;
+	data: string;
+	serverOrder: number;
+	replicaId: string;
+}
+export type OperationHistoryRow = Selectable<OperationHistoryTable>;
+
+export interface DocumentBaselineTable {
+	oid: string;
+	snapshot: string;
+	timestamp: string;
+}
+export type DocumentBaselineRow = Selectable<DocumentBaselineTable>;
+
+export interface ReplicaInfoTable {
+	id: string;
+	clientId: string;
+	lastSeenWallClockTime: number | null;
+	ackedLogicalTime: string | null;
+	type: ReplicaType;
+	ackedServerOrder: Generated<number>;
+}
+
+export type ReplicaInfoRow = Selectable<ReplicaInfoTable>;
+
+export interface FileMetadataTable {
+	fileId: string;
+	name: string;
+	type: string;
+	pendingDeleteAt: number | null;
+}
+
+export type FileMetadataRow = Selectable<FileMetadataTable>;

package/src/storage/sqlShard/transfer.test.ts

@@ -0,0 +1,130 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { openDatabase as openUnifiedDatabase } from '../sql/database.js';
+import {
+	randomBaselines,
+	randomFileMetadata,
+	randomOperations,
+	randomReplicaInfo,
+} from './_testData/unifiedData.js';
+import { join, dirname } from 'path';
+import { transferToShards } from './transfer.js';
+import { rm, mkdir } from 'fs/promises';
+import { fileURLToPath } from 'url';
+import {
+	DocumentBaselineRow,
+	FileMetadataRow,
+	OperationHistoryRow,
+	ReplicaInfoRow,
+} from '../sql/tables.js';
+import { Kysely } from 'kysely';
+
+const testTempDir = join(
+	dirname(fileURLToPath(import.meta.url)),
+	'_testData',
+	'tmp',
+);
+
+describe('sql-shard storage transfer utility', () => {
+	const databases = new Array<Kysely<any>>();
+	beforeAll(async () => {
+		try {
+			await rm(testTempDir, { recursive: true });
+		} catch (err) {}
+		// create a temp dir
+		await mkdir(testTempDir, { recursive: true });
+	});
+	afterAll(async () => {
+		await Promise.all(databases.map((db) => db.destroy()));
+		// empty the temp dir
+		await rm(testTempDir, { recursive: true });
+	});
+
+	it('should transfer data from unified database to shards', async () => {
+		const unifiedFile = join(testTempDir, 'unified.sqlite');
+		const { db: unifiedDb, ready } = openUnifiedDatabase(unifiedFile);
+		databases.push(unifiedDb);
+		await ready;
+		const libraryIds = ['library-1', 'library-2', 'library-3'];
+		const randomData = {} as Record<
+			string,
+			{
+				operations: OperationHistoryRow[];
+				baselines: DocumentBaselineRow[];
+				replicas: ReplicaInfoRow[];
+				fileMetadata: FileMetadataRow[];
+			}
+		>;
+		for (const libraryId of libraryIds) {
+			randomData[libraryId] = {
+				operations: randomOperations(libraryId),
+				baselines: randomBaselines(libraryId),
+				replicas: randomReplicaInfo(libraryId),
+				fileMetadata: randomFileMetadata(libraryId),
+			};
+			await unifiedDb
+				.insertInto('ReplicaInfo')
+				.values(randomData[libraryId].replicas)
+				.execute();
+			await unifiedDb
+				.insertInto('OperationHistory')
+				.values(randomData[libraryId].operations)
+				.execute();
+			await unifiedDb
+				.insertInto('DocumentBaseline')
+				.values(randomData[libraryId].baselines)
+				.execute();
+			await unifiedDb
+				.insertInto('FileMetadata')
+				.values(randomData[libraryId].fileMetadata)
+				.execute();
+		}
+
+		const shards = await transferToShards({
+			directory: testTempDir,
+			file: unifiedFile,
+		});
+		Object.values(shards).map((db) => databases.push(db));
+
+		expect(Object.keys(shards).sort()).toEqual(libraryIds.sort());
+
+		for (const [libraryId, db] of Object.entries(shards)) {
+			const operations = await db
+				.selectFrom('OperationHistory')
+				.selectAll()
+				.execute();
+			const baselines = await db
+				.selectFrom('DocumentBaseline')
+				.selectAll()
+				.execute();
+			const replicas = await db.selectFrom('ReplicaInfo').selectAll().execute();
+			const fileMetadata = await db
+				.selectFrom('FileMetadata')
+				.selectAll()
+				.execute();
+
+			expect(operations.length).toBe(randomData[libraryId].operations.length);
+			expect(baselines.length).toBe(randomData[libraryId].baselines.length);
+			expect(replicas.length).toBe(randomData[libraryId].replicas.length);
+			expect(fileMetadata.length).toBe(
+				randomData[libraryId].fileMetadata.length,
+			);
+
+			for (const { oid } of randomData[libraryId].operations) {
+				expect(operations.find((op) => op.oid === oid)).toBeDefined();
+			}
+			for (const { oid } of randomData[libraryId].baselines) {
+				expect(
+					baselines.find((baseline) => baseline.oid === oid),
+				).toBeDefined();
+			}
+			for (const { id } of randomData[libraryId].replicas) {
+				expect(replicas.find((replica) => replica.id === id)).toBeDefined();
+			}
+			for (const { fileId } of randomData[libraryId].fileMetadata) {
+				expect(
+					fileMetadata.find((file) => file.fileId === fileId),
+				).toBeDefined();
+			}
+		}
+	});
+});