@verdant-web/server 2.1.3 → 3.0.0

package/src/storage/sql/SqlReplicas.ts

@@ -0,0 +1,234 @@
+import { StoredReplicaInfo } from '../../types.js';
+import { ReplicaStorage } from '../Storage.js';
+import { sql, type Kysely } from 'kysely';
+import { Database } from './tables.js';
+import { ReplicaType, VerdantError } from '@verdant-web/common';
+
+export class SqlReplicas implements ReplicaStorage {
+	constructor(
+		private db: Kysely<Database>,
+		private readonly replicaTruancyMinutes: number,
+		private readonly dialect: 'sqlite' | 'postgres',
+	) {}
+
+	get truantCutoff(): number {
+		return Date.now() - this.replicaTruancyMinutes * 60 * 1000;
+	}
+
+	get = async (
+		libraryId: string,
+		replicaId: string,
+	): Promise<StoredReplicaInfo | null> => {
+		return (
+			(await this.db
+				.selectFrom('ReplicaInfo')
+				.where('libraryId', '=', libraryId)
+				.where('id', '=', replicaId)
+				.selectAll()
+				.executeTakeFirst()) ?? null
+		);
+	};
+	getOrCreate = async (
+		libraryId: string,
+		replicaId: string,
+		info: { userId: string; type: ReplicaType },
+	): Promise<{
+		status: 'new' | 'existing' | 'truant';
+		replicaInfo: StoredReplicaInfo;
+	}> => {
+		const existing = await this.get(libraryId, replicaId);
+		if (!existing) {
+			const created = await this.db
+				.insertInto('ReplicaInfo')
+				.values({
+					id: replicaId,
+					clientId: info.userId,
+					libraryId,
+					type: info.type,
+					ackedServerOrder: 0,
+				})
+				.returningAll()
+				.executeTakeFirst();
+			if (!created) {
+				throw new VerdantError(
+					VerdantError.Code.Unexpected,
+					undefined,
+					'Failed to create replica',
+				);
+			}
+			return { status: 'new', replicaInfo: created };
+		}
+
+		if (existing.type !== info.type) {
+			// type should be updated if a new token changes it
+			await this.db
+				.updateTable('ReplicaInfo')
+				.set({ type: info.type })
+				.where('libraryId', '=', libraryId)
+				.where('id', '=', replicaId)
+				.execute();
+		}
+
+		if (existing.clientId !== info.userId) {
+			// replicas cannot change hands - this is a security issue
+			throw new VerdantError(
+				VerdantError.Code.Forbidden,
+				undefined,
+				'Another user is already using this replica ID',
+			);
+		}
+
+		if (
+			existing.lastSeenWallClockTime !== null &&
+			existing.lastSeenWallClockTime < this.truantCutoff
+		) {
+			return { status: 'truant', replicaInfo: existing };
+		}
+
+		return { status: 'existing', replicaInfo: existing };
+	};
+
+	getAll = async (
+		libraryId: string,
+		options?: { omitTruant: boolean } | undefined,
+	): Promise<StoredReplicaInfo[]> => {
+		let builder = this.db
+			.selectFrom('ReplicaInfo')
+			.where('libraryId', '=', libraryId);
+
+		if (options?.omitTruant) {
+			builder = builder.where('lastSeenWallClockTime', '>', this.truantCutoff);
+		}
+
+		return builder.selectAll().execute();
+	};
+
+	updateLastSeen = async (
+		libraryId: string,
+		replicaId: string,
+	): Promise<void> => {
+		const clockTime = Date.now();
+		await this.db
+			.updateTable('ReplicaInfo')
+			.set({ lastSeenWallClockTime: clockTime })
+			.where('libraryId', '=', libraryId)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	updateAckedServerOrder = async (
+		libraryId: string,
+		replicaId: string,
+		serverOrder: number,
+	): Promise<void> => {
+		const max = this.dialect === 'postgres' ? 'GREATEST' : 'MAX';
+		await this.db
+			.updateTable('ReplicaInfo')
+			.set(
+				'ackedServerOrder',
+				({ val }) =>
+					sql<number>`${sql.raw(max)}(ackedServerOrder, ${val(serverOrder)})`,
+			)
+			.where('libraryId', '=', libraryId)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	updateAcknowledgedLogicalTime = async (
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	): Promise<void> => {
+		await this.db
+			.updateTable('ReplicaInfo')
+			.set({ ackedLogicalTime: timestamp })
+			.where('libraryId', '=', libraryId)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	getEarliestAckedServerOrder = async (libraryId: string): Promise<number> => {
+		// gets earliest acked server order of all non-truant replicas.
+		const res = await this.db
+			.selectFrom('ReplicaInfo')
+			.where('libraryId', '=', libraryId)
+			.where('lastSeenWallClockTime', '>', this.truantCutoff)
+			.orderBy('ackedServerOrder', 'asc')
+			.select('ackedServerOrder')
+			.executeTakeFirst();
+		return res?.ackedServerOrder ?? 0;
+	};
+
+	acknowledgeOperation = async (
+		libraryId: string,
+		replicaId: string,
+		timestamp: string,
+	): Promise<void> => {
+		if (!timestamp) return;
+		// when acking an operation, we also set the replica's server order
+		// to that operation's server order, if it's greater.
+		const max = this.dialect === 'postgres' ? 'GREATEST' : 'MAX';
+		await this.db
+			.updateTable('ReplicaInfo')
+			.set({ ackedLogicalTime: timestamp })
+			.set(
+				'ackedServerOrder',
+				({ val }) => sql<number>`${sql.raw(max)}(
+					ackedServerOrder,
+					COALESCE(
+						(
+        			SELECT serverOrder FROM OperationHistory
+        			WHERE libraryId = ${val(libraryId)} AND timestamp = ${val(timestamp)}
+      			),
+						0
+					)
+				)`,
+			)
+			.where('libraryId', '=', libraryId)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+
+	getGlobalAck = async (
+		libraryId: string,
+		onlineReplicaIds?: string[] | undefined,
+	): Promise<string | null> => {
+		const nonTruant = await this.getAll(libraryId, { omitTruant: true });
+		if (nonTruant.length === 0) return null;
+		const globalAckEligible = nonTruant.filter(
+			(replica) => replica.type < 2 || onlineReplicaIds?.includes(replica.id),
+		);
+
+		return globalAckEligible.reduce(
+			(acc, replica) => {
+				if (!replica.ackedLogicalTime) return acc;
+				if (acc === null) return replica.ackedLogicalTime;
+				return acc < replica.ackedLogicalTime ? acc : replica.ackedLogicalTime;
+			},
+			null as string | null,
+		);
+	};
+	delete = async (libraryId: string, replicaId: string): Promise<void> => {
+		await this.db
+			.deleteFrom('ReplicaInfo')
+			.where('libraryId', '=', libraryId)
+			.where('id', '=', replicaId)
+			.execute();
+	};
+	deleteAll = async (libraryId: string): Promise<void> => {
+		await this.db
+			.deleteFrom('ReplicaInfo')
+			.where('libraryId', '=', libraryId)
+			.execute();
+	};
+	deleteAllForUser = async (
+		libraryId: string,
+		userId: string,
+	): Promise<void> => {
+		await this.db
+			.deleteFrom('ReplicaInfo')
+			.where('libraryId', '=', libraryId)
+			.where('clientId', '=', userId)
+			.execute();
+	};
+}

package/src/storage/sql/migrations/v1.ts

@@ -0,0 +1,64 @@
+import { Kysely } from 'kysely';
+
+export async function up(db: Kysely<any>) {
+	await db.schema
+		.createTable('DocumentBaseline')
+		.ifNotExists()
+		.addColumn('oid', 'text')
+		.addColumn('snapshot', 'text')
+		.addColumn('timestamp', 'text', (cb) => cb.notNull())
+		.addColumn('libraryId', 'text', (cb) => cb.notNull())
+		.addPrimaryKeyConstraint('DocumentBaseline_primaryKey', [
+			'libraryId',
+			'oid',
+		])
+		.execute();
+
+	await db.schema
+		.createTable('OperationHistory')
+		.ifNotExists()
+		.addColumn('oid', 'text', (cb) => cb.notNull())
+		.addColumn('timestamp', 'text', (cb) => cb.notNull())
+		.addColumn('data', 'text', (cb) => cb.notNull())
+		.addColumn('serverOrder', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.addColumn('replicaId', 'text', (cb) => cb.notNull())
+		.addColumn('libraryId', 'text', (cb) => cb.notNull())
+		.addPrimaryKeyConstraint('OperationHistory_primaryKey', [
+			'libraryId',
+			'replicaId',
+			'oid',
+			'timestamp',
+		])
+		.execute();
+
+	await db.schema
+		.createTable('ReplicaInfo')
+		.ifNotExists()
+		.addColumn('id', 'text')
+		.addColumn('libraryId', 'text', (cb) => cb.notNull())
+		.addColumn('clientId', 'text', (cb) => cb.notNull())
+		.addColumn('lastSeenWallClockTime', 'integer')
+		.addColumn('ackedLogicalTime', 'text')
+		.addColumn('type', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.addColumn('ackedServerOrder', 'integer', (cb) => cb.notNull().defaultTo(0))
+		.addPrimaryKeyConstraint('ReplicaInfo_primaryKey', ['libraryId', 'id'])
+		.execute();
+
+	await db.schema
+		.createTable('FileMetadata')
+		.ifNotExists()
+		.addColumn('libraryId', 'text', (cb) => cb.notNull())
+		.addColumn('fileId', 'text', (cb) => cb.notNull())
+		.addColumn('name', 'text', (cb) => cb.notNull())
+		.addColumn('type', 'text', (cb) => cb.notNull())
+		.addColumn('pendingDeleteAt', 'integer')
+		.addPrimaryKeyConstraint('FileMetadata_primaryKey', ['libraryId', 'fileId'])
+		.execute();
+}
+
+export async function down(db: Kysely<any>) {
+	await db.schema.dropTable('DocumentBaseline').execute();
+	await db.schema.dropTable('OperationHistory').execute();
+	await db.schema.dropTable('ReplicaInfo').execute();
+	await db.schema.dropTable('FileMetadata').execute();
+}

package/src/storage/sql/migrations.ts

@@ -0,0 +1,3 @@
+import * as v1 from './migrations/v1.js';
+
+export default { v1 };

package/src/storage/sql/sqlStorage.ts

@@ -0,0 +1,59 @@
+import Database from 'better-sqlite3';
+import { Kysely, SqliteDialect } from 'kysely';
+import { StorageFactory } from '../Storage.js';
+import { SqlBaselines } from './SqlBaselines.js';
+import { SqlOperations } from './SqlOperations.js';
+import { SqlReplicas } from './SqlReplicas.js';
+import { Database as DatabaseTypes } from './tables.js';
+import { SqlFileMetadata } from './SqlFileMetadata.js';
+import { migrateToLatest } from '@a-type/kysely';
+import migrations from './migrations.js';
+
+export const sqlStorage = ({
+	databaseFile: dbFile,
+	skipMigrations,
+}: {
+	databaseFile: string;
+	skipMigrations?: boolean;
+}): StorageFactory => {
+	const db = new Kysely<DatabaseTypes>({
+		dialect: new SqliteDialect({
+			database: new Database(dbFile),
+		}),
+	});
+	let ready = Promise.resolve<void>(undefined);
+	if (!skipMigrations) {
+		ready = migrateToLatest(db, migrations);
+	}
+	return (options) => {
+		const baselines = new SqlBaselines(db, 'sqlite');
+		const operations = new SqlOperations(db, 'sqlite');
+		const replicas = new SqlReplicas(
+			db,
+			options.replicaTruancyMinutes,
+			'sqlite',
+		);
+		const fileMetadata = new SqlFileMetadata(
+			db,
+			options.fileDeleteExpirationDays,
+			'sqlite',
+		);
+		const close = async () => {
+			storage.open = false;
+			await db.destroy();
+		};
+		const storage = {
+			baselines,
+			operations,
+			replicas,
+			fileMetadata,
+			close,
+			open: false,
+			ready,
+		};
+		ready.then(() => {
+			storage.open = true;
+		});
+		return storage;
+	};
+};

package/src/storage/sql/tables.ts

@@ -0,0 +1,45 @@
+import { ReplicaType } from '@verdant-web/common';
+import { Generated, Selectable } from 'kysely';
+
+export interface Database {
+	OperationHistory: OperationHistoryTable;
+	DocumentBaseline: DocumentBaselineTable;
+	ReplicaInfo: ReplicaInfoTable;
+	FileMetadata: FileMetadataTable;
+}
+
+export interface OperationHistoryTable {
+	oid: string;
+	timestamp: string;
+	data: string;
+	serverOrder: number;
+	replicaId: string;
+	libraryId: string;
+}
+export type OperationHistoryRow = Selectable<OperationHistoryTable>;
+
+export interface DocumentBaselineTable {
+	oid: string;
+	snapshot: string;
+	timestamp: string;
+	libraryId: string;
+}
+export type DocumentBaselineRow = Selectable<DocumentBaselineTable>;
+
+export interface ReplicaInfoTable {
+	id: string;
+	libraryId: string;
+	clientId: string;
+	lastSeenWallClockTime: number | null;
+	ackedLogicalTime: string | null;
+	type: ReplicaType;
+	ackedServerOrder: Generated<number>;
+}
+
+export interface FileMetadataTable {
+	libraryId: string;
+	fileId: string;
+	name: string;
+	type: string;
+	pendingDeleteAt: number | null;
+}

package/src/types.ts

@@ -5,19 +5,22 @@ import {
 	ReplicaType,
 } from '@verdant-web/common';
 
-export interface OperationSpec extends Operation {
+export interface StoredOperation extends Operation {
 	serverOrder: number;
 	replicaId: string;
 }
 
-export interface DocumentBaselineSpec
+export interface StoredDocumentBaseline
 	extends Omit<DocumentBaseline<any>, 'snapshot'> {
 	snapshot: string;
 	libraryId: string;
-	serverOrder: number;
 }
 
-export interface ReplicaInfoSpec extends ReplicaInfo {
+export interface HydratedDocumentBaseline extends DocumentBaseline<any> {
+	libraryId: string;
+}
+
+export interface StoredReplicaInfo extends ReplicaInfo {
 	libraryId: string;
 	// the authenticated client ID authorized
 	// to write to this replica
@@ -43,6 +46,14 @@ export type LibraryInfo<Profile = unknown> = {
 	globalAck: string | null;
 };
 
+export interface FileMetadata {
+	libraryId: string;
+	fileId: string;
+	name: string;
+	type: string;
+	pendingDeleteAt: number | null;
+}
+
 // patch File
 declare global {
 	interface File {

package/bin/server.mjs

@@ -1,115 +0,0 @@
-#!/usr/bin/env node
-import { createServer } from 'http';
-import { createReadStream } from 'fs';
-import { parse, URLSearchParams } from 'url';
-import {
-	LocalFileStorage,
-	Server,
-	TokenProvider,
-	ReplicaType,
-} from '../dist/esm/index.js';
-import minimist from 'minimist';
-
-const argv = minimist(process.argv.slice(1));
-
-const PORT = argv.port || argv.p || process.env.VERDANT_PORT || 3242;
-const SECRET =
-	argv.secret || process.env.VERDANT_SECRET || 'notsecretnotsecretnotsecret';
-
-const tokenProvider = new TokenProvider({ secret: SECRET });
-
-const http = createServer((req, res) => {
-	const url = parse(req.url);
-	// write cors headers for incoming host
-	const origin = req.headers.origin;
-	res.setHeader('Access-Control-Allow-Origin', origin ?? '*');
-	res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
-	res.setHeader(
-		'Access-Control-Allow-Headers',
-		'Content-Type, Authorization, X-Request-Id',
-	);
-	res.setHeader('Access-Control-Allow-Credentials', 'true');
-	res.setHeader('Access-Control-Max-Age', '86400');
-	if (req.method === 'OPTIONS') {
-		res.writeHead(200);
-		res.end();
-		return;
-	}
-
-	// serve files on the /files path
-	if (url.pathname.startsWith('/files/')) {
-		const fileStream = createReadStream(`files/${url.pathname.slice(7)}`);
-		fileStream.pipe(res);
-		fileStream.on('error', (err) => {
-			res.writeHead(404);
-			res.end();
-		});
-		return;
-	}
-
-	// provide tokens on the /auth path
-	if (url.pathname.startsWith('/auth/')) {
-		// userId is provided as a query param
-		const params = new URLSearchParams(url.search);
-		const userId = params.get('userId');
-
-		// library ID is in path
-		const libraryId = url.pathname.slice(6);
-
-		const token = tokenProvider.getToken({
-			userId,
-			libraryId,
-			syncEndpoint: `http://localhost:${PORT}/sync`,
-			role: 'user',
-			type: ReplicaType.Realtime,
-		});
-		res.writeHead(200, {
-			'Content-Type': 'application/json',
-		});
-		res.end(JSON.stringify({ accessToken: token }));
-		return;
-	} else if (url.pathname.startsWith('/sync')) {
-		if (url.pathname.startsWith('/sync/files/')) {
-			return verdant.handleFileRequest(req, res);
-		}
-		return verdant.handleRequest(req, res);
-	} else {
-		res.writeHead(404);
-		res.end();
-	}
-});
-
-const verdant = new Server({
-	databaseFile: 'verdant.sqlite',
-	tokenSecret: SECRET,
-	fileStorage: new LocalFileStorage({
-		rootDirectory: 'files',
-		host: `http://localhost:${3242}/files`,
-	}),
-	profiles: {
-		get: (userId) => ({
-			id: userId,
-		}),
-	},
-	httpServer: http,
-	log: console.log,
-});
-
-verdant.on('error', (err) => {
-	console.error(err);
-});
-
-http.listen(PORT, () => {
-	console.log(`🌿 Verdant standalone server listening on port ${PORT}`);
-	console.log(``);
-	console.log(
-		`It's not recommended you use this server in production. It's a convenient way to test syncing locally, but has no authorization.`,
-	);
-	console.log(``);
-	console.log(
-		`To connect your client, supply "http://localhost:${PORT}/auth/<libraryId>?userId=<userId>" to sync.authEndpoint.`,
-	);
-	console.log(
-		`<libraryId> and <userId> are up to you and your app. Clients on the same library ID will sync data.`,
-	);
-});

package/dist/cjs/Baselines.d.ts

@@ -1,16 +0,0 @@
-import { DocumentBaseline, Operation, Ref } from '@verdant-web/common';
-import { Database } from 'better-sqlite3';
-export declare class Baselines {
-    private db;
-    constructor(db: Database);
-    get: (libraryId: string, oid: string) => DocumentBaseline<any> | null;
-    getAllWithSubObjects: (libraryId: string, oid: string) => DocumentBaseline<any>[];
-    set: (libraryId: string, baseline: DocumentBaseline) => import("better-sqlite3").RunResult;
-    insertAll: (libraryId: string, baselines: DocumentBaseline[]) => void;
-    private hydrateSnapshot;
-    getAllAfter: (libraryId: string, timestamp: string | null) => DocumentBaseline<any>[];
-    getFromServerOrder: (libraryId: string, serverOrder: number) => DocumentBaseline<any>[];
-    applyOperations: (libraryId: string, oid: string, operations: Operation[], deletedRefs?: Ref[]) => never[] | undefined;
-    deleteAll: (libraryId: string) => void;
-    getCount: (libraryId: string) => number;
-}