@verdant-web/server 3.3.10 → 4.0.0

package/src/libraries/generic.ts

@@ -0,0 +1,9 @@
+import { LibraryInfo } from '../types.js';
+import { LibraryFileInfo } from './Library.js';
+
+export interface LibraryApi {
+	getFileInfo(fileId: string): Promise<LibraryFileInfo | null>;
+	forceTruant: (replicaId: string) => Promise<void>;
+	getInfo: () => Promise<LibraryInfo | null>;
+	getDocumentSnapshot: (collection: string, id: string) => any;
+}

package/src/libraries/singleNode.ts

@@ -0,0 +1,145 @@
+import {
+	ClientMessage,
+	createOid,
+	EventSubscriber,
+	VerdantError,
+} from '@verdant-web/common';
+import { Readable } from 'stream';
+import { ReadableStream } from 'stream/web';
+import { UserProfileLoader } from '../Profiles.js';
+import { TokenInfo, TokenVerifier } from '../TokenVerifier.js';
+import { ClientConnectionManager } from '../connections/ClientConnection.js';
+import {
+	FileInfo,
+	FileStorage,
+	FileStorageLibraryDelegate,
+} from '../files/FileStorage.js';
+import { Library } from '../internals.js';
+import { Logger } from '../logger.js';
+import { StorageFactory } from '../storage/Storage.js';
+import { LibraryEvents } from './Library.js';
+import { LibraryApi } from './generic.js';
+
+export interface SingleNodeMicroserverConfig {
+	storage: StorageFactory;
+	fileStorage?: FileStorage;
+	disableRebasing?: boolean;
+	profiles: UserProfileLoader<any>;
+	tokenSecret: string;
+	log?: Logger;
+	__testMode?: boolean;
+}
+
+const noop = () => {};
+
+export class SingleNodeLibraryManager {
+	public readonly events = new EventSubscriber<LibraryEvents>();
+	private microserverPromises: Map<string, Promise<SingleNodeMicroserver>> =
+		new Map();
+	public readonly tokenVerifier: TokenVerifier;
+
+	constructor(private ctx: SingleNodeMicroserverConfig) {
+		this.tokenVerifier = new TokenVerifier({ secret: ctx.tokenSecret });
+	}
+
+	public get log() {
+		return this.ctx.log ?? noop;
+	}
+
+	public get __testMode() {
+		return this.ctx.__testMode === true;
+	}
+
+	get = async (libraryId: string) => {
+		let microserverPromise = this.microserverPromises.get(libraryId);
+		if (!microserverPromise) {
+			microserverPromise = this.constructMicroserver(libraryId);
+			this.microserverPromises.set(libraryId, microserverPromise);
+		}
+		return microserverPromise;
+	};
+
+	evict = async (libraryId: string) => {
+		const lib = await this.get(libraryId);
+		await lib.destroy();
+		this.microserverPromises.delete(libraryId);
+	};
+
+	private constructMicroserver = async (libraryId: string) => {
+		const clientConnections = new ClientConnectionManager({
+			profiles: this.ctx.profiles,
+		});
+		const storage = await this.ctx.storage(libraryId);
+		const library = new Library({
+			storage,
+			id: libraryId,
+			events: this.events,
+			sender: clientConnections,
+			presence: clientConnections.presence,
+			log: (level, ...args) =>
+				this.ctx.log?.(level, `[Library ${libraryId}]`, ...args),
+			fileStorage: this.ctx.fileStorage
+				? new FileStorageLibraryDelegate(libraryId, this.ctx.fileStorage)
+				: undefined,
+			disableRebasing: this.ctx.disableRebasing || false,
+		});
+		return new SingleNodeMicroserver(library, clientConnections, this.ctx);
+	};
+}
+
+export class SingleNodeMicroserver implements LibraryApi {
+	constructor(
+		private library: Library,
+		public readonly clientConnections: ClientConnectionManager,
+		private ctx: SingleNodeMicroserverConfig,
+	) {}
+
+	handleMessage = async (
+		key: string,
+		info: TokenInfo,
+		message: ClientMessage,
+	) => {
+		return this.library.handleMessage(message, key, info);
+	};
+
+	uploadFile = async (
+		fileStream: Readable | ReadableStream | any,
+		info: FileInfo,
+	) => {
+		if (!this.ctx.fileStorage) {
+			throw new VerdantError(
+				VerdantError.Code.ConfigurationError,
+				undefined,
+				'File storage not configured',
+			);
+		}
+		await this.library.putFileInfo(info);
+		await this.ctx.fileStorage.put(fileStream, info);
+	};
+
+	getFileInfo = async (fileId: string) => {
+		return this.library.getFileInfo(fileId);
+	};
+
+	onRequest = this.clientConnections.addFetch.bind(this.clientConnections);
+
+	getDocumentSnapshot = async (
+		collection: string,
+		documentId: string,
+	): Promise<any | null> => {
+		return this.library.getDocumentSnapshot(createOid(collection, documentId));
+	};
+
+	destroy = async () => {
+		this.clientConnections.disconnectAll();
+		await this.library.destroy();
+	};
+
+	getInfo = async () => {
+		return this.library.getInfo();
+	};
+
+	forceTruant = async (replicaId: string) => {
+		await this.library.forceTruant(replicaId);
+	};
+}

package/src/logger.ts

@@ -0,0 +1,4 @@
+export type Logger = (
+	level: 'info' | 'warn' | 'error' | 'debug',
+	...args: any[]
+) => void;

package/src/storage/Storage.ts

@@ -4,87 +4,58 @@ import {
 	Ref,
 	ReplicaType,
 } from '@verdant-web/common';
+import { FileInfo } from '../files/FileStorage.js';
 import {
+	FileMetadata,
 	HydratedDocumentBaseline,
 	StoredOperation,
 	StoredReplicaInfo,
-	FileMetadata,
 } from '../types.js';
-import { FileInfo } from '../files/FileStorage.js';
 
 export interface ReplicaStorage {
 	readonly truantCutoff: number;
-	get(libraryId: string, replicaId: string): Promise<StoredReplicaInfo | null>;
+	get(replicaId: string): Promise<StoredReplicaInfo | null>;
 	getOrCreate(
-		libraryId: string,
 		replicaId: string,
 		info: { userId: string; type: ReplicaType },
 	): Promise<{
 		status: 'new' | 'existing' | 'truant';
 		replicaInfo: StoredReplicaInfo;
 	}>;
-	getAll(
-		libraryId: string,
-		options?: { omitTruant: boolean },
-	): Promise<StoredReplicaInfo[]>;
-	updateLastSeen(libraryId: string, replicaId: string): Promise<void>;
-	updateAckedServerOrder(
-		libraryId: string,
-		replicaId: string,
-		serverOrder: number,
-	): Promise<void>;
+	getAll(options?: { omitTruant: boolean }): Promise<StoredReplicaInfo[]>;
+	updateLastSeen(replicaId: string): Promise<void>;
+	updateAckedServerOrder(replicaId: string, serverOrder: number): Promise<void>;
 	updateAcknowledgedLogicalTime(
-		libraryId: string,
 		replicaId: string,
 		timestamp: string,
 	): Promise<void>;
-	getEarliestAckedServerOrder(libraryId: string): Promise<number>;
-	acknowledgeOperation(
-		libraryId: string,
-		replicaId: string,
-		timestamp: string,
-	): Promise<void>;
-	getGlobalAck(
-		libraryId: string,
-		onlineReplicaIds?: string[],
-	): Promise<string | null>;
-	delete(libraryId: string, replicaId: string): Promise<void>;
-	deleteAll(libraryId: string): Promise<void>;
-	deleteAllForUser(libraryId: string, userId: string): Promise<void>;
+	getEarliestAckedServerOrder(): Promise<number>;
+	acknowledgeOperation(replicaId: string, timestamp: string): Promise<void>;
+	getGlobalAck(onlineReplicaIds?: string[]): Promise<string | null>;
+	delete(replicaId: string): Promise<void>;
+	deleteAll(): Promise<void>;
+	deleteAllForUser(userId: string): Promise<void>;
+	forceTruant(replicaId: string): Promise<void>;
 }
 
 export interface OperationStorage {
-	getAll(libraryId: string, oid: string): Promise<StoredOperation[]>;
-	getBeforeServerOrder(
-		libraryId: string,
-		beforeServerOrder: number,
-	): Promise<StoredOperation[]>;
-	getAfterServerOrder(
-		libraryId: string,
-		afterServerOrder: number,
-	): Promise<StoredOperation[]>;
-	getLatestServerOrder(libraryId: string): Promise<number>;
-	getCount(libraryId: string): Promise<number>;
-	insertAll(
-		libraryId: string,
-		replicaId: string,
-		operations: Operation[],
-	): Promise<number>;
-	deleteAll(libraryId: string): Promise<void>;
-	delete(libraryId: string, operations: Operation[]): Promise<void>;
+	getAll(oid: string): Promise<StoredOperation[]>;
+	getBeforeServerOrder(beforeServerOrder: number): Promise<StoredOperation[]>;
+	getAfterServerOrder(afterServerOrder: number): Promise<StoredOperation[]>;
+	getLatestServerOrder(): Promise<number>;
+	getCount(): Promise<number>;
+	insertAll(replicaId: string, operations: Operation[]): Promise<number>;
+	deleteAll(): Promise<void>;
+	delete(operations: Operation[]): Promise<void>;
 }
 
 export interface BaselineStorage {
-	get(libraryId: string, oid: string): Promise<HydratedDocumentBaseline | null>;
-	getAll(libraryId: string): Promise<HydratedDocumentBaseline[]>;
-	insertAll(
-		libraryId: string,
-		baselines: DocumentBaseline<any>[],
-	): Promise<void>;
-	deleteAll(libraryId: string): Promise<void>;
-	getCount(libraryId: string): Promise<number>;
+	get(oid: string): Promise<HydratedDocumentBaseline | null>;
+	getAll(): Promise<HydratedDocumentBaseline[]>;
+	insertAll(baselines: DocumentBaseline<any>[]): Promise<void>;
+	deleteAll(): Promise<void>;
+	getCount(): Promise<number>;
 	applyOperations(
-		libraryId: string,
 		oid: string,
 		operations: StoredOperation[],
 		deletedRefs?: Ref[],
@@ -92,18 +63,18 @@ export interface BaselineStorage {
 }
 
 export interface FileMetadataStorage {
-	get(libraryId: string, fileId: string): Promise<FileMetadata | null>;
-	getAll(libraryId: string): Promise<FileMetadata[]>;
-	deleteAll(libraryId: string): Promise<void>;
-	put(libraryId: string, fileInfo: FileInfo): Promise<void>;
-	markPendingDelete(libraryId: string, fileId: string): Promise<void>;
-	delete(libraryId: string, fileId: string): Promise<void>;
-	getPendingDelete(libraryId: string): Promise<FileMetadata[]>;
+	get(fileId: string): Promise<FileMetadata | null>;
+	getAll(): Promise<FileMetadata[]>;
+	deleteAll(): Promise<void>;
+	put(fileInfo: FileInfo): Promise<void>;
+	markPendingDelete(fileId: string): Promise<void>;
+	delete(fileId: string): Promise<void>;
+	getPendingDelete(): Promise<FileMetadata[]>;
 }
 
 export interface StorageOptions {
-	replicaTruancyMinutes: number;
-	fileDeleteExpirationDays: number;
+	replicaTruancyMinutes?: number;
+	fileDeleteExpirationDays?: number;
 }
 export interface Storage {
 	replicas: ReplicaStorage;
@@ -112,7 +83,7 @@ export interface Storage {
 	fileMetadata: FileMetadataStorage;
 	close(): Promise<void>;
 	readonly open: boolean;
-	readonly ready: Promise<void>;
 }
 
-export type StorageFactory = (options: StorageOptions) => Storage;
+export type StorageFactory = (libraryId: string) => Promise<Storage>;
+export type StorageFactoryFactory = (options: StorageOptions) => StorageFactory;

package/src/storage/StorageManager.ts

@@ -0,0 +1,39 @@
+import { Storage, StorageFactory } from './Storage.js';
+
+export class StorageManager {
+	private _cache: Map<string, Promise<Storage>> = new Map();
+	private _evicts: Map<string, NodeJS.Timeout> = new Map();
+
+	constructor(private _factory: StorageFactory) {}
+
+	open = async (libraryId: string) => {
+		let storage = this._cache.get(libraryId);
+		if (!storage) {
+			storage = this._factory(libraryId);
+			this._cache.set(libraryId, storage);
+		}
+		this.refresh(libraryId);
+		await storage;
+		return storage;
+	};
+
+	closeAll = async () => {
+		for (const [libraryId, storage] of this._cache) {
+			await (await storage).close();
+			this.evict(libraryId);
+		}
+	};
+
+	private refresh = (libraryId: string) => {
+		clearTimeout(this._evicts.get(libraryId));
+		this._evicts.set(
+			libraryId,
+			setTimeout(this.evict, 1000 * 60 * 60, libraryId),
+		);
+	};
+
+	private evict = (libraryId: string) => {
+		this._cache.delete(libraryId);
+		this._evicts.delete(libraryId);
+	};
+}

package/src/storage/index.ts

@@ -1,3 +1,2 @@
-export * from './Storage.js';
-export * from './sql/sqlStorage.js';
 export * from './sqlShard/sqlShardStorage.js';
+export * from './Storage.js';

package/src/storage/sqlShard/Databases.ts

@@ -1,23 +1,28 @@
-import { Kysely } from 'kysely';
-import { openDatabase } from './database.js';
-import { Database as DatabaseTypes } from './tables.js';
-import { mkdirSync, existsSync } from 'fs';
 import { assert } from '@verdant-web/common';
+import { existsSync, mkdirSync } from 'fs';
+import { Logger } from '../../logger.js';
+import {
+	createFilesystemExecutor,
+	openDatabase,
+	SqliteExecutor,
+} from './database.js';
 
 export class Databases {
-	private openPromises: Record<string, Promise<Kysely<DatabaseTypes>>> = {};
-	private cache: Record<string, Kysely<DatabaseTypes>> = {};
+	private openPromises: Record<string, Promise<SqliteExecutor>> = {};
+	private cache: Record<string, SqliteExecutor> = {};
 	private closeTimeouts: Record<string, NodeJS.Timeout> = {};
 	private openedPreviously: Record<string, boolean> = {};
 	private directory;
 	private closeTimeout;
 	private disableWal = false;
-	private closed = new WeakMap<Kysely<DatabaseTypes>, boolean>();
+	private closed = new WeakMap<SqliteExecutor, boolean>();
+	private log: Logger;
 
 	constructor(config: {
 		directory: string;
 		closeTimeout?: number;
 		disableWal?: boolean;
+		log?: Logger;
 	}) {
 		this.directory = config.directory;
 		this.closeTimeout = config.closeTimeout ?? 1000 * 60 * 60;
@@ -25,9 +30,10 @@ export class Databases {
 		if (this.directory !== ':memory:' && !existsSync(this.directory)) {
 			mkdirSync(this.directory, { recursive: true });
 		}
+		this.log = config.log ?? (() => {});
 	}
 
-	get = async (libraryId: string): Promise<Kysely<DatabaseTypes>> => {
+	get = async (libraryId: string): Promise<SqliteExecutor> => {
 		let db = this.cache[libraryId];
 		if (db) {
 			if (!this.closed.get(db)) {
@@ -37,10 +43,17 @@ export class Databases {
 		}
 		let openPromise = this.openPromises[libraryId];
 		if (!openPromise) {
-			this.openPromises[libraryId] = openDatabase(this.directory, libraryId, {
-				skipMigrations: this.openedPreviously[libraryId],
-				disableWal: this.disableWal,
-			});
+			// only a promise for potential "forward compatibility"
+			// but tbh I'm not sure a promise-based storage solution
+			// is on the horizon. SQLite is fine.
+			this.openPromises[libraryId] = Promise.resolve(
+				openDatabase(
+					createFilesystemExecutor(this.directory, libraryId, {
+						disableWal: this.disableWal,
+						log: this.log,
+					}),
+				),
+			);
 			openPromise = this.openPromises[libraryId];
 		}
 		db = await openPromise;
@@ -68,7 +81,7 @@ export class Databases {
 		if (db) {
 			this.closed.set(db, true);
 			delete this.cache[libraryId];
-			db.destroy();
+			db.close();
 		}
 	};
 

package/src/storage/sqlShard/SqlBaselines.ts

@@ -1,87 +1,82 @@
-import { Kysely } from 'kysely';
-import { BaselineStorage } from '../Storage.js';
-import { Database, DocumentBaselineRow } from './tables.js';
 import { DocumentBaseline, Ref, applyPatch } from '@verdant-web/common';
 import { HydratedDocumentBaseline, StoredOperation } from '../../types.js';
-import { Databases } from './Databases.js';
+import { BaselineStorage } from '../Storage.js';
+import { SqliteExecutor } from './database.js';
+import { DocumentBaselineRow } from './tables.js';
 
 export class SqlBaselines implements BaselineStorage {
 	constructor(
-		private dbs: Databases,
-		private dialect: 'postgres' | 'sqlite',
+		private db: SqliteExecutor,
+		private libraryId: string,
 	) {}
 
 	get = async (
-		libraryId: string,
 		oid: string,
-		{ tx }: { tx?: Kysely<Database> } = {},
+		{ tx }: { tx?: SqliteExecutor } = {},
 	): Promise<HydratedDocumentBaseline | null> => {
-		const db = tx ?? (await this.dbs.get(libraryId));
-		const raw =
-			(await db
-				.selectFrom('DocumentBaseline')
-				.where('oid', '=', oid)
-				.selectAll()
-				.executeTakeFirst()) ?? null;
+		return this.#getSync(oid, { tx });
+	};
+	#getSync = (oid: string, { tx }: { tx?: SqliteExecutor } = {}) => {
+		const db = tx ?? this.db;
+		const raw = db.first<DocumentBaselineRow>(
+			`
+					SELECT * FROM DocumentBaseline
+					WHERE oid = ?
+				`,
+			[oid],
+		);
 		if (!raw) return null;
 		this.hydrate(raw);
 		return raw as any;
 	};
 
-	getAll = async (libraryId: string) => {
-		const db = await this.dbs.get(libraryId);
-		const raw = await db
-			.selectFrom('DocumentBaseline')
-			.orderBy('timestamp', 'asc')
-			.selectAll()
-			.execute();
+	getAll = async () => {
+		const db = this.db;
+		const raw = db.query<DocumentBaselineRow>(
+			`SELECT * FROM DocumentBaseline ORDER BY timestamp ASC`,
+		);
 
 		raw.forEach(this.hydrate);
 
 		return raw as any;
 	};
 
-	set = async (
-		libraryId: string,
+	set = (
 		baseline: DocumentBaseline,
 		{
 			tx,
 		}: {
-			tx?: Kysely<Database>;
+			tx?: SqliteExecutor;
 		},
 	) => {
-		const db = tx ?? (await this.dbs.get(libraryId));
+		const db = tx ?? this.db;
 		if (!baseline.snapshot) {
-			await db
-				.deleteFrom('DocumentBaseline')
-				.where('oid', '=', baseline.oid)
-				.execute();
-			return;
+			return db.exec(`DELETE FROM DocumentBaseline WHERE oid = ?`, [
+				baseline.oid,
+			]);
 		}
 
-		await db
-			.insertInto('DocumentBaseline')
-			.values({
-				oid: baseline.oid,
-				snapshot: JSON.stringify(baseline.snapshot),
-				timestamp: baseline.timestamp,
-				authz: baseline.authz,
-			})
-			.onConflict((cb) =>
-				cb.doUpdateSet({
-					snapshot: JSON.stringify(baseline.snapshot),
-					timestamp: baseline.timestamp,
-					authz: baseline.authz,
-				}),
-			)
-			.execute();
+		return db.exec(
+			`INSERT INTO DocumentBaseline (oid, snapshot, timestamp, authz) VALUES (?, ?, ?, ?)
+				ON CONFLICT(oid) DO UPDATE SET
+					snapshot = excluded.snapshot,
+					timestamp = excluded.timestamp,
+					authz = excluded.authz
+				`,
+			[
+				baseline.oid,
+				JSON.stringify(baseline.snapshot),
+				baseline.timestamp,
+				baseline.authz,
+			],
+		);
 	};
 
-	insertAll = async (libraryId: string, baselines: DocumentBaseline[]) => {
-		const db = await this.dbs.get(libraryId);
-		await db.transaction().execute(async (tx) => {
+	insertAll = async (baselines: DocumentBaseline[]) => {
+		const db = this.db;
+		return db.transaction((tx) => {
 			for (const baseline of baselines) {
-				await this.set(libraryId, baseline, { tx });
+				this.set(baseline, { tx });
 			}
 		});
 	};
@@ -92,23 +87,22 @@ export class SqlBaselines implements BaselineStorage {
 	};
 
 	applyOperations = async (
-		libraryId: string,
 		oid: string,
 		operations: StoredOperation[],
 		deletedRefs?: Ref[],
 	) => {
 		if (operations.length === 0) return;
 
-		const db = await this.dbs.get(libraryId);
+		const db = this.db;
 
-		await db.transaction().execute(async (tx) => {
-			let baseline = await this.get(libraryId, oid, { tx });
+		await db.transaction((tx) => {
+			let baseline = this.#getSync(oid, { tx });
 			if (!baseline) {
 				baseline = {
 					oid,
 					snapshot: {},
 					timestamp: operations[0].timestamp,
-					libraryId,
+					libraryId: this.libraryId,
 					authz: operations[0].authz,
 				};
 			}
@@ -123,23 +117,20 @@ export class SqlBaselines implements BaselineStorage {
 				}
 			}
 			baseline.timestamp = operations[operations.length - 1].timestamp;
-			await this.set(libraryId, baseline, { tx });
+			this.set(baseline, { tx });
 		});
 	};
 
-	deleteAll = async (libraryId: string) => {
-		const db = await this.dbs.get(libraryId);
-		await db.deleteFrom('DocumentBaseline').execute();
+	deleteAll = async () => {
+		const db = this.db;
+		db.exec('DELETE FROM DocumentBaseline;');
 	};
 
-	getCount = async (libraryId: string) => {
-		const db = await this.dbs.get(libraryId);
+	getCount = async () => {
+		const db = this.db;
 		return (
-			(
-				await db
-					.selectFrom('DocumentBaseline')
-					.select(({ fn }) => fn.countAll<number>().as('count'))
-					.executeTakeFirst()
+			db.first<{ count: number }>(
+				`SELECT COUNT(*) as count FROM DocumentBaseline;`,
 			)?.count ?? 0
 		);
 	};