@verdant-web/server 2.1.3 → 3.0.0

package/dist/esm/storage/sql/sqlStorage.js

@@ -0,0 +1,43 @@
+import Database from 'better-sqlite3';
+import { Kysely, SqliteDialect } from 'kysely';
+import { SqlBaselines } from './SqlBaselines.js';
+import { SqlOperations } from './SqlOperations.js';
+import { SqlReplicas } from './SqlReplicas.js';
+import { SqlFileMetadata } from './SqlFileMetadata.js';
+import { migrateToLatest } from '@a-type/kysely';
+import migrations from './migrations.js';
+export const sqlStorage = ({ databaseFile: dbFile, skipMigrations, }) => {
+    const db = new Kysely({
+        dialect: new SqliteDialect({
+            database: new Database(dbFile),
+        }),
+    });
+    let ready = Promise.resolve(undefined);
+    if (!skipMigrations) {
+        ready = migrateToLatest(db, migrations);
+    }
+    return (options) => {
+        const baselines = new SqlBaselines(db, 'sqlite');
+        const operations = new SqlOperations(db, 'sqlite');
+        const replicas = new SqlReplicas(db, options.replicaTruancyMinutes, 'sqlite');
+        const fileMetadata = new SqlFileMetadata(db, options.fileDeleteExpirationDays, 'sqlite');
+        const close = async () => {
+            storage.open = false;
+            await db.destroy();
+        };
+        const storage = {
+            baselines,
+            operations,
+            replicas,
+            fileMetadata,
+            close,
+            open: false,
+            ready,
+        };
+        ready.then(() => {
+            storage.open = true;
+        });
+        return storage;
+    };
+};
+//# sourceMappingURL=sqlStorage.js.map

package/dist/esm/storage/sql/sqlStorage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlStorage.js","sourceRoot":"","sources":["../../../../src/storage/sql/sqlStorage.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAE/C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,UAAU,MAAM,iBAAiB,CAAC;AAEzC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAC1B,YAAY,EAAE,MAAM,EACpB,cAAc,GAId,EAAkB,EAAE;IACpB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAgB;QACpC,OAAO,EAAE,IAAI,aAAa,CAAC;YAC1B,QAAQ,EAAE,IAAI,QAAQ,CAAC,MAAM,CAAC;SAC9B,CAAC;KACF,CAAC,CAAC;IACH,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAO,SAAS,CAAC,CAAC;IAC7C,IAAI,CAAC,cAAc,EAAE,CAAC;QACrB,KAAK,GAAG,eAAe,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,CAAC,OAAO,EAAE,EAAE;QAClB,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,WAAW,CAC/B,EAAE,EACF,OAAO,CAAC,qBAAqB,EAC7B,QAAQ,CACR,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,eAAe,CACvC,EAAE,EACF,OAAO,CAAC,wBAAwB,EAChC,QAAQ,CACR,CAAC;QACF,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC;YACrB,MAAM,EAAE,CAAC,OAAO,EAAE,CAAC;QACpB,CAAC,CAAC;QACF,MAAM,OAAO,GAAG;YACf,SAAS;YACT,UAAU;YACV,QAAQ;YACR,YAAY;YACZ,KAAK;YACL,IAAI,EAAE,KAAK;YACX,KAAK;SACL,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IAChB,CAAC,CAAC;AACH,CAAC,CAAC"}

package/dist/esm/storage/sql/tables.d.ts

@@ -0,0 +1,40 @@
+import { ReplicaType } from '@verdant-web/common';
+import { Generated, Selectable } from 'kysely';
+export interface Database {
+    OperationHistory: OperationHistoryTable;
+    DocumentBaseline: DocumentBaselineTable;
+    ReplicaInfo: ReplicaInfoTable;
+    FileMetadata: FileMetadataTable;
+}
+export interface OperationHistoryTable {
+    oid: string;
+    timestamp: string;
+    data: string;
+    serverOrder: number;
+    replicaId: string;
+    libraryId: string;
+}
+export type OperationHistoryRow = Selectable<OperationHistoryTable>;
+export interface DocumentBaselineTable {
+    oid: string;
+    snapshot: string;
+    timestamp: string;
+    libraryId: string;
+}
+export type DocumentBaselineRow = Selectable<DocumentBaselineTable>;
+export interface ReplicaInfoTable {
+    id: string;
+    libraryId: string;
+    clientId: string;
+    lastSeenWallClockTime: number | null;
+    ackedLogicalTime: string | null;
+    type: ReplicaType;
+    ackedServerOrder: Generated<number>;
+}
+export interface FileMetadataTable {
+    libraryId: string;
+    fileId: string;
+    name: string;
+    type: string;
+    pendingDeleteAt: number | null;
+}

package/dist/esm/storage/sql/tables.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tables.js.map

package/dist/esm/storage/sql/tables.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tables.js","sourceRoot":"","sources":["../../../../src/storage/sql/tables.ts"],"names":[],"mappings":""}

package/dist/esm/types.d.ts

@@ -1,21 +1,23 @@
 import { DocumentBaseline, Operation, ReplicaInfo, ReplicaType } from '@verdant-web/common';
-export interface OperationSpec extends Operation {
+export interface StoredOperation extends Operation {
     serverOrder: number;
     replicaId: string;
 }
-export interface DocumentBaselineSpec extends Omit<DocumentBaseline<any>, 'snapshot'> {
+export interface StoredDocumentBaseline extends Omit<DocumentBaseline<any>, 'snapshot'> {
     snapshot: string;
     libraryId: string;
-    serverOrder: number;
 }
-export interface ReplicaInfoSpec extends ReplicaInfo {
+export interface HydratedDocumentBaseline extends DocumentBaseline<any> {
+    libraryId: string;
+}
+export interface StoredReplicaInfo extends ReplicaInfo {
     libraryId: string;
     clientId: string;
     lastSeenWallClockTime: number | null;
     type: ReplicaType;
     ackedServerOrder: number;
 }
-export declare type LibraryInfo<Profile = unknown> = {
+export type LibraryInfo<Profile = unknown> = {
     id: string;
     replicas: {
         id: string;
@@ -30,6 +32,13 @@ export declare type LibraryInfo<Profile = unknown> = {
     baselinesCount: number;
     globalAck: string | null;
 };
+export interface FileMetadata {
+    libraryId: string;
+    fileId: string;
+    name: string;
+    type: string;
+    pendingDeleteAt: number | null;
+}
 declare global {
     interface File {
         readonly lastModified: number;

package/package.json

@@ -1,13 +1,21 @@
 {
   "name": "@verdant-web/server",
-  "version": "2.1.3",
+  "version": "3.0.0",
   "access": "public",
   "type": "module",
-  "main": "./dist/cjs/index.js",
-  "module": "./dist/esm/index.js",
   "bin": {
     "verdant-server": "./bin/server.mjs"
   },
+  "exports": {
+    ".": {
+      "import": "./dist/esm/index.js",
+      "types": "./dist/esm/index.d.ts"
+    },
+    "./storage": {
+      "import": "./dist/esm/storage/index.js",
+      "types": "./dist/esm/storage/index.d.ts"
+    }
+  },
   "publishConfig": {
     "access": "public"
   },
@@ -16,13 +24,16 @@
     "src/"
   ],
   "dependencies": {
+    "@a-type/kysely": "^1.0.1",
     "async-lock": "^1.4.0",
     "better-sqlite3": "^7.6.2",
     "busboy": "^1.6.0",
     "jsonwebtoken": "^8.5.1",
+    "kysely": "0.27.3",
+    "kysely-plugin-serialize": "^0.5.7",
     "minimist": "^1.2.8",
     "ws": "^8.9.0",
-    "@verdant-web/common": "2.2.0"
+    "@verdant-web/common": "2.3.0"
   },
   "devDependencies": {
     "@types/async-lock": "^1.3.0",
@@ -31,20 +42,15 @@
     "@types/jsonwebtoken": "^8.5.9",
     "@types/node": "^20.11.5",
     "@types/ws": "^8.5.3",
-    "typescript": "^4.7.4",
+    "cross-env": "7.0.3",
+    "typescript": "^5.4.2",
     "vitest": "^1.1.0"
   },
   "scripts": {
-    "build": "tsc -p tsconfig.json && tsc -p tsconfig-cjs.json",
+    "build": "tsc",
     "prepublish": "pnpm run build",
     "link": "pnpm link --global",
-    "test": "vitest"
-  },
-  "types": "./dist/esm/index.d.ts",
-  "exports": {
-    ".": {
-      "import": "./dist/esm/index.js",
-      "require": "./dist/cjs/index.js"
-    }
+    "test": "vitest",
+    "perf": "pnpm run build && cross-env VERDANT_LOG=false node ./perf/perf.mjs"
   }
 }

package/src/Server.ts

@@ -9,28 +9,21 @@ import {
 	ServerMessage,
 	VerdantError,
 } from '@verdant-web/common';
+import busboy from 'busboy';
 import EventEmitter from 'events';
-import { IncomingMessage, Server as HttpServer, ServerResponse } from 'http';
-import { WebSocketServer, WebSocket } from 'ws';
-import { UserProfileLoader, UserProfiles } from './Profiles.js';
-import create from 'better-sqlite3';
-import { MessageSender } from './MessageSender.js';
+import { Server as HttpServer, IncomingMessage, ServerResponse } from 'http';
+import { TransformStream } from 'node:stream/web';
+import internal, { Readable } from 'stream';
 import { URL } from 'url';
+import { WebSocket, WebSocketServer } from 'ws';
 import { ClientConnectionManager } from './ClientConnection.js';
-import { ReplicaKeepaliveTimers } from './ReplicaKeepaliveTimers.js';
-import { TokenInfo, TokenVerifier } from './TokenVerifier.js';
-import busboy from 'busboy';
 import { FileInfo, FileStorage } from './files/FileStorage.js';
-import internal, { Readable } from 'stream';
-import { FileMetadata, FileMetadataConfig } from './files/FileMetadata.js';
+import { MessageSender } from './MessageSender.js';
+import { UserProfileLoader, UserProfiles } from './Profiles.js';
+import { ReplicaKeepaliveTimers } from './ReplicaKeepaliveTimers.js';
 import { ServerLibrary } from './ServerLibrary.js';
-import { migrations } from './migrations.js';
-import {
-	ReadableStream,
-	ReadableWritablePair,
-	TransformStream,
-	WritableStream,
-} from 'node:stream/web';
+import { TokenInfo, TokenVerifier } from './TokenVerifier.js';
+import { Storage, StorageFactory } from './storage/Storage.js';
 
 export interface ServerOptions {
 	/**
@@ -45,9 +38,10 @@ export interface ServerOptions {
 	 */
 	tokenSecret: string;
 	/**
-	 * The path to your SQLite database.
+	 * Your storage implementation. See the sqlStorage export for a built-in
+	 * option.
 	 */
-	databaseFile: string;
+	storage: StorageFactory;
 	/**
 	 * Optionally provide an implementation of the UserProfiles interface to look up
 	 * static user profile data to use in persistence. This data is considered trusted
@@ -76,7 +70,7 @@ export interface ServerOptions {
 	 * of this library.
 	 */
 	fileStorage?: FileStorage;
-	fileConfig?: FileMetadataConfig;
+	fileConfig?: { deleteExpirationDays?: number };
 }
 
 class DefaultProfiles implements UserProfiles<{ id: string }> {
@@ -117,7 +111,7 @@ export class Server extends EventEmitter implements MessageSender {
 	private httpServer!: HttpServer;
 	private wss: WebSocketServer;
 	private fileStorage?: FileStorage;
-	private fileMetadata;
+	private storage: Storage;
 	private library;
 
 	private clientConnections = new ClientConnectionManager();
@@ -144,23 +138,20 @@ export class Server extends EventEmitter implements MessageSender {
 		});
 
 		this.fileStorage = options.fileStorage;
-
-		const db = create(options.databaseFile);
-		migrations(db);
-
-		this.fileMetadata = new FileMetadata(db, options.fileConfig);
+		this.storage = options.storage({
+			fileDeleteExpirationDays: options.fileConfig?.deleteExpirationDays ?? 14,
+			replicaTruancyMinutes: options.replicaTruancyMinutes || 60 * 24 * 14,
+		});
 
 		this.library = new ServerLibrary({
-			db,
+			storage: this.storage,
 			sender: this,
 			profiles: new UserProfileLoader(
 				options.profiles || new DefaultProfiles(),
 			),
 			log: this.log,
 			disableRebasing: options.disableRebasing,
-			fileMetadata: this.fileMetadata,
 			fileStorage: this.fileStorage,
-			replicaTruancyMinutes: options.replicaTruancyMinutes || 60 * 24 * 14,
 		});
 
 		this.library.subscribe(
@@ -396,7 +387,7 @@ export class Server extends EventEmitter implements MessageSender {
 
 	private writeErrorResponse(e: unknown, res: ServerResponse) {
 		this.emit('error', e);
-		this.log('Error handling request', e);
+		this.log('error', 'Error handling request', e);
 
 		if (e instanceof VerdantError) {
 			// write error data to response
@@ -416,7 +407,7 @@ export class Server extends EventEmitter implements MessageSender {
 	// for fetch-style
 	private getErrorResponse(e: unknown) {
 		this.emit('error', e);
-		this.log('Error handling request', e);
+		this.log('error', 'Error handling request', e);
 
 		if (e instanceof VerdantError) {
 			return new Response(JSON.stringify(e.toResponse()), {
@@ -467,7 +458,7 @@ export class Server extends EventEmitter implements MessageSender {
 					headers: req.headers,
 					id,
 				});
-				this.log('File upload complete');
+				this.log('info', 'File upload complete', id);
 				res.writeHead(200);
 				res.write(JSON.stringify({ success: true }));
 				res.end();
@@ -525,7 +516,7 @@ export class Server extends EventEmitter implements MessageSender {
 					headers: headersAsRecord,
 					id,
 				});
-				this.log('File upload complete');
+				this.log('info', 'File upload complete', id);
 				return new Response(JSON.stringify({ success: true }), {
 					status: 200,
 					headers: {
@@ -571,7 +562,7 @@ export class Server extends EventEmitter implements MessageSender {
 		return new Promise((resolve, reject) => {
 			const bb = busboy({ headers });
 
-			bb.on('file', (fieldName, stream, fileInfo) => {
+			bb.on('file', async (fieldName, stream, fileInfo) => {
 				// too many 'info's....
 				const lofiFileInfo: FileInfo = {
 					id,
@@ -581,7 +572,8 @@ export class Server extends EventEmitter implements MessageSender {
 				};
 				// write metadata to storage
 				try {
-					this.fileMetadata.put(info.libraryId, lofiFileInfo);
+					await this.storage.ready;
+					await this.storage.fileMetadata.put(info.libraryId, lofiFileInfo);
 					fs.put(stream, lofiFileInfo);
 				} catch (e) {
 					reject(e);
@@ -620,7 +612,8 @@ export class Server extends EventEmitter implements MessageSender {
 	): Promise<FileData> => {
 		const fs = this.getFileStorageOrThrow();
 
-		const fileInfo = this.fileMetadata.get(info.libraryId, id);
+		await this.storage.ready;
+		const fileInfo = await this.storage.fileMetadata.get(info.libraryId, id);
 		if (!fileInfo) {
 			throw new VerdantError(
 				VerdantError.Code.NotFound,
@@ -663,6 +656,7 @@ export class Server extends EventEmitter implements MessageSender {
 		message: ClientMessage,
 	) => {
 		this.log(
+			'debug',
 			'Got message from user',
 			info.userId,
 			', library',
@@ -715,26 +709,26 @@ export class Server extends EventEmitter implements MessageSender {
 		await Promise.all([
 			new Promise<void>((resolve) => {
 				setTimeout(() => {
-					console.warn('HTTP server close timed out');
+					this.log('warn', 'HTTP server close timed out');
 					resolve();
 				}, 5 * 1000);
 				this.httpServer.close(() => {
 					resolve();
-					this.log('HTTP server closed');
+					this.log('info', 'HTTP server closed');
 				});
 			}),
 			new Promise<void>((resolve) => {
 				setTimeout(() => {
-					console.warn('Socket server close timed out');
+					this.log('warn', 'Socket server close timed out');
 					resolve();
 				}, 5 * 1000);
 				this.wss.close(() => {
 					resolve();
-					this.log('Socket server closed');
+					this.log('info', 'Socket server closed');
 				});
 			}),
 		]);
-		this.library.close();
+		await this.storage.close();
 	};
 
 	/**
@@ -749,7 +743,7 @@ export class Server extends EventEmitter implements MessageSender {
 		// disconnect all clients
 		this.clientConnections.disconnectAll(libraryId);
 		this.library.destroy(libraryId);
-		this.log('Evicted library', libraryId);
+		this.log('info', 'Evicted library', libraryId);
 	};
 
 	/**
@@ -775,7 +769,7 @@ export class Server extends EventEmitter implements MessageSender {
 	 */
 	evictUser = (libraryId: string, userId: string) => {
 		this.library.evictUser(libraryId, userId);
-		this.log('Evicted user', userId, 'from library', libraryId);
+		this.log('info', 'Evicted user', userId, 'from library', libraryId);
 	};
 
 	/**