@verdant-web/server 3.0.4 → 3.1.0

package/dist/esm/storage/sqlShard/transfer.test.js

@@ -0,0 +1,94 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { openDatabase as openUnifiedDatabase } from '../sql/database.js';
+import { randomBaselines, randomFileMetadata, randomOperations, randomReplicaInfo, } from './_testData/unifiedData.js';
+import { join, dirname } from 'path';
+import { transferToShards } from './transfer.js';
+import { rm, mkdir } from 'fs/promises';
+import { fileURLToPath } from 'url';
+const testTempDir = join(dirname(fileURLToPath(import.meta.url)), '_testData', 'tmp');
+describe('sql-shard storage transfer utility', () => {
+    const databases = new Array();
+    beforeAll(async () => {
+        try {
+            await rm(testTempDir, { recursive: true });
+        }
+        catch (err) { }
+        // create a temp dir
+        await mkdir(testTempDir, { recursive: true });
+    });
+    afterAll(async () => {
+        await Promise.all(databases.map((db) => db.destroy()));
+        // empty the temp dir
+        await rm(testTempDir, { recursive: true });
+    });
+    it('should transfer data from unified database to shards', async () => {
+        const unifiedFile = join(testTempDir, 'unified.sqlite');
+        const { db: unifiedDb, ready } = openUnifiedDatabase(unifiedFile);
+        databases.push(unifiedDb);
+        await ready;
+        const libraryIds = ['library-1', 'library-2', 'library-3'];
+        const randomData = {};
+        for (const libraryId of libraryIds) {
+            randomData[libraryId] = {
+                operations: randomOperations(libraryId),
+                baselines: randomBaselines(libraryId),
+                replicas: randomReplicaInfo(libraryId),
+                fileMetadata: randomFileMetadata(libraryId),
+            };
+            await unifiedDb
+                .insertInto('ReplicaInfo')
+                .values(randomData[libraryId].replicas)
+                .execute();
+            await unifiedDb
+                .insertInto('OperationHistory')
+                .values(randomData[libraryId].operations)
+                .execute();
+            await unifiedDb
+                .insertInto('DocumentBaseline')
+                .values(randomData[libraryId].baselines)
+                .execute();
+            await unifiedDb
+                .insertInto('FileMetadata')
+                .values(randomData[libraryId].fileMetadata)
+                .execute();
+        }
+        const shards = await transferToShards({
+            directory: testTempDir,
+            file: unifiedFile,
+        });
+        Object.values(shards).map((db) => databases.push(db));
+        expect(Object.keys(shards).sort()).toEqual(libraryIds.sort());
+        for (const [libraryId, db] of Object.entries(shards)) {
+            const operations = await db
+                .selectFrom('OperationHistory')
+                .selectAll()
+                .execute();
+            const baselines = await db
+                .selectFrom('DocumentBaseline')
+                .selectAll()
+                .execute();
+            const replicas = await db.selectFrom('ReplicaInfo').selectAll().execute();
+            const fileMetadata = await db
+                .selectFrom('FileMetadata')
+                .selectAll()
+                .execute();
+            expect(operations.length).toBe(randomData[libraryId].operations.length);
+            expect(baselines.length).toBe(randomData[libraryId].baselines.length);
+            expect(replicas.length).toBe(randomData[libraryId].replicas.length);
+            expect(fileMetadata.length).toBe(randomData[libraryId].fileMetadata.length);
+            for (const { oid } of randomData[libraryId].operations) {
+                expect(operations.find((op) => op.oid === oid)).toBeDefined();
+            }
+            for (const { oid } of randomData[libraryId].baselines) {
+                expect(baselines.find((baseline) => baseline.oid === oid)).toBeDefined();
+            }
+            for (const { id } of randomData[libraryId].replicas) {
+                expect(replicas.find((replica) => replica.id === id)).toBeDefined();
+            }
+            for (const { fileId } of randomData[libraryId].fileMetadata) {
+                expect(fileMetadata.find((file) => file.fileId === fileId)).toBeDefined();
+            }
+        }
+    });
+});
+//# sourceMappingURL=transfer.test.js.map

package/dist/esm/storage/sqlShard/transfer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transfer.test.js","sourceRoot":"","sources":["../../../../src/storage/sqlShard/transfer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACnE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EACN,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GACjB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AASpC,MAAM,WAAW,GAAG,IAAI,CACvB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACvC,WAAW,EACX,KAAK,CACL,CAAC;AAEF,QAAQ,CAAC,oCAAoC,EAAE,GAAG,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,KAAK,EAAe,CAAC;IAC3C,SAAS,CAAC,KAAK,IAAI,EAAE;QACpB,IAAI,CAAC;YACJ,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAA,CAAC;QAChB,oBAAoB;QACpB,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACvD,qBAAqB;QACrB,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;QACxD,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAClE,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1B,MAAM,KAAK,CAAC;QACZ,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,EAQlB,CAAC;QACF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,UAAU,CAAC,SAAS,CAAC,GAAG;gBACvB,UAAU,EAAE,gBAAgB,CAAC,SAAS,CAAC;gBACvC,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC;gBACrC,QAAQ,EAAE,iBAAiB,CAAC,SAAS,CAAC;gBACtC,YAAY,EAAE,kBAAkB,CAAC,SAAS,CAAC;aAC3C,CAAC;YACF,MAAM,SAAS;iBACb,UAAU,CAAC,aAAa,CAAC;iBACzB,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;iBACtC,OAAO,EAAE,CAAC;YACZ,MAAM,SAAS;iBACb,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC;iBACxC,OAAO,EAAE,CAAC;YACZ,MAAM,SAAS;iBACb,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC;iBACvC,OAAO,EAAE,CAAC;YACZ,MAAM,SAAS;iBACb,UAAU,CAAC,cAAc,CAAC;iBAC1B,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC;iBAC1C,OAAO,EAAE,CAAC;QACb,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC;YACrC,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,WAAW;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAE9D,KAAK,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,MAAM,UAAU,GAAG,MAAM,EAAE;iBACzB,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YACZ,MAAM,SAAS,GAAG,MAAM,EAAE;iBACxB,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,CAAC;YAC1E,MAAM,YAAY,GAAG,MAAM,EAAE;iBAC3B,UAAU,CAAC,cAAc,CAAC;iBAC1B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YAEZ,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACxE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACtE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,CAC/B,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,MAAM,CACzC,CAAC;YAEF,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE,CAAC;gBACxD,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/D,CAAC;YACD,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,EAAE,CAAC;gBACvD,MAAM,CACL,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,KAAK,GAAG,CAAC,CAClD,CAAC,WAAW,EAAE,CAAC;YACjB,CAAC;YACD,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACrD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACrE,CAAC;YACD,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,EAAE,CAAC;gBAC7D,MAAM,CACL,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,CACnD,CAAC,WAAW,EAAE,CAAC;YACjB,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@verdant-web/server",
-  "version": "3.0.4",
+  "version": "3.1.0",
   "access": "public",
   "type": "module",
   "bin": {
@@ -33,7 +33,7 @@
     "kysely-plugin-serialize": "^0.5.7",
     "minimist": "^1.2.8",
     "ws": "^8.9.0",
-    "@verdant-web/common": "2.3.3"
+    "@verdant-web/common": "2.3.4"
   },
   "devDependencies": {
     "@types/async-lock": "^1.3.0",
@@ -51,6 +51,7 @@
     "prepublish": "pnpm run build",
     "link": "pnpm link --global",
     "test": "vitest",
+    "ci:test:unit": "vitest run",
     "perf": "pnpm run build && cross-env VERDANT_LOG=false node ./perf/perf.mjs"
   }
 }

package/src/ServerLibrary.test.ts

@@ -143,6 +143,16 @@ describe('ServerLibrary', () => {
 				'library-1',
 				{
 					type: 'op-re',
+					baselines: [
+						{
+							oid: 'notes/1',
+							timestamp: `faketime-0`,
+							snapshot: {
+								title: 'My Note',
+								content: 'This is my note.',
+							},
+						},
+					],
 					operations: [
 						{
 							oid: 'notes/1',

package/src/storage/index.ts

@@ -1,2 +1,3 @@
 export * from './Storage.js';
 export * from './sql/sqlStorage.js';
+export * from './sqlShard/sqlShardStorage.js';

package/src/storage/sql/database.ts

@@ -0,0 +1,32 @@
+import { Kysely, SqliteDialect } from 'kysely';
+import { Database as DatabaseTypes } from './tables.js';
+import Database from 'better-sqlite3';
+import { migrateToLatest } from '@a-type/kysely';
+import migrations from './migrations.js';
+
+export function openDatabase(
+	file: string,
+	options: {
+		skipMigrations?: boolean;
+		disableWal?: boolean;
+	} = {},
+) {
+	const internalDb = new Database(file);
+	if (!options.disableWal) {
+		internalDb.pragma('journal_mode = WAL');
+	}
+	const db = new Kysely<DatabaseTypes>({
+		dialect: new SqliteDialect({
+			database: internalDb,
+		}),
+	});
+	let ready = Promise.resolve<void>(undefined);
+	if (!options.skipMigrations) {
+		ready = migrateToLatest(db, migrations);
+	}
+
+	return {
+		db,
+		ready,
+	};
+}

package/src/storage/sql/sqlStorage.ts

@@ -8,23 +8,21 @@ import { Database as DatabaseTypes } from './tables.js';
 import { SqlFileMetadata } from './SqlFileMetadata.js';
 import { migrateToLatest } from '@a-type/kysely';
 import migrations from './migrations.js';
+import { openDatabase } from './database.js';
 
 export const sqlStorage = ({
 	databaseFile: dbFile,
 	skipMigrations,
+	disableWal,
 }: {
 	databaseFile: string;
 	skipMigrations?: boolean;
+	disableWal?: boolean;
 }): StorageFactory => {
-	const db = new Kysely<DatabaseTypes>({
-		dialect: new SqliteDialect({
-			database: new Database(dbFile),
-		}),
+	const { db, ready } = openDatabase(dbFile, {
+		skipMigrations,
+		disableWal,
 	});
-	let ready = Promise.resolve<void>(undefined);
-	if (!skipMigrations) {
-		ready = migrateToLatest(db, migrations);
-	}
 	return (options) => {
 		const baselines = new SqlBaselines(db, 'sqlite');
 		const operations = new SqlOperations(db, 'sqlite');

package/src/storage/sql/tables.ts

@@ -36,6 +36,8 @@ export interface ReplicaInfoTable {
 	ackedServerOrder: Generated<number>;
 }
 
+export type ReplicaInfoRow = Selectable<ReplicaInfoTable>;
+
 export interface FileMetadataTable {
 	libraryId: string;
 	fileId: string;
@@ -43,3 +45,5 @@ export interface FileMetadataTable {
 	type: string;
 	pendingDeleteAt: number | null;
 }
+
+export type FileMetadataRow = Selectable<FileMetadataTable>;

package/src/storage/sqlShard/Databases.ts

@@ -0,0 +1,80 @@
+import { Kysely } from 'kysely';
+import { openDatabase } from './database.js';
+import { Database as DatabaseTypes } from './tables.js';
+import { mkdirSync, existsSync } from 'fs';
+import { assert } from '@verdant-web/common';
+
+export class Databases {
+	private openPromises: Record<string, Promise<Kysely<DatabaseTypes>>> = {};
+	private cache: Record<string, Kysely<DatabaseTypes>> = {};
+	private closeTimeouts: Record<string, NodeJS.Timeout> = {};
+	private openedPreviously: Record<string, boolean> = {};
+	private directory;
+	private closeTimeout;
+	private disableWal = false;
+	private closed = new WeakMap<Kysely<DatabaseTypes>, boolean>();
+
+	constructor(config: {
+		directory: string;
+		closeTimeout?: number;
+		disableWal?: boolean;
+	}) {
+		this.directory = config.directory;
+		this.closeTimeout = config.closeTimeout ?? 1000 * 60 * 60;
+		this.disableWal = config.disableWal ?? false;
+		if (this.directory !== ':memory:' && !existsSync(this.directory)) {
+			mkdirSync(this.directory, { recursive: true });
+		}
+	}
+
+	get = async (libraryId: string): Promise<Kysely<DatabaseTypes>> => {
+		let db = this.cache[libraryId];
+		if (db) {
+			if (!this.closed.get(db)) {
+				this.enqueueClose(libraryId);
+				return this.cache[libraryId];
+			}
+		}
+		let openPromise = this.openPromises[libraryId];
+		if (!openPromise) {
+			this.openPromises[libraryId] = openDatabase(this.directory, libraryId, {
+				skipMigrations: this.openedPreviously[libraryId],
+				disableWal: this.disableWal,
+			});
+			openPromise = this.openPromises[libraryId];
+		}
+		db = await openPromise;
+		this.openedPreviously[libraryId] = true;
+		this.cache[libraryId] = db;
+		delete this.openPromises[libraryId];
+		this.enqueueClose(libraryId);
+		assert(db, `Database was not loaded: ${libraryId}`);
+		return db;
+	};
+
+	private enqueueClose = (libraryId: string) => {
+		clearTimeout(this.closeTimeouts[libraryId]);
+		this.closeTimeouts[libraryId] = setTimeout(
+			this.close,
+			this.closeTimeout,
+			libraryId,
+		);
+	};
+
+	close = (libraryId: string) => {
+		clearTimeout(this.closeTimeouts[libraryId]);
+		delete this.openPromises[libraryId];
+		const db = this.cache[libraryId];
+		if (db) {
+			this.closed.set(db, true);
+			delete this.cache[libraryId];
+			db.destroy();
+		}
+	};
+
+	destroy = async () => {
+		for (const libraryId in this.cache) {
+			this.close(libraryId);
+		}
+	};
+}

package/src/storage/sqlShard/SqlBaselines.ts

@@ -0,0 +1,140 @@
+import { Kysely } from 'kysely';
+import { BaselineStorage } from '../Storage.js';
+import { Database, DocumentBaselineRow } from './tables.js';
+import { DocumentBaseline, Ref, applyPatch } from '@verdant-web/common';
+import { HydratedDocumentBaseline, StoredOperation } from '../../types.js';
+import { Databases } from './Databases.js';
+
+export class SqlBaselines implements BaselineStorage {
+	constructor(
+		private dbs: Databases,
+		private dialect: 'postgres' | 'sqlite',
+	) {}
+
+	get = async (
+		libraryId: string,
+		oid: string,
+		{ tx }: { tx?: Kysely<Database> } = {},
+	): Promise<HydratedDocumentBaseline | null> => {
+		const db = tx ?? (await this.dbs.get(libraryId));
+		const raw =
+			(await db
+				.selectFrom('DocumentBaseline')
+				.where('oid', '=', oid)
+				.selectAll()
+				.executeTakeFirst()) ?? null;
+		if (!raw) return null;
+		this.hydrate(raw);
+		return raw as any;
+	};
+
+	getAll = async (libraryId: string) => {
+		const db = await this.dbs.get(libraryId);
+		const raw = await db
+			.selectFrom('DocumentBaseline')
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+
+		return raw as any;
+	};
+
+	set = async (
+		libraryId: string,
+		baseline: DocumentBaseline,
+		{
+			tx,
+		}: {
+			tx?: Kysely<Database>;
+		},
+	) => {
+		const db = tx ?? (await this.dbs.get(libraryId));
+		if (!baseline.snapshot) {
+			await db
+				.deleteFrom('DocumentBaseline')
+				.where('oid', '=', baseline.oid)
+				.execute();
+			return;
+		}
+
+		await db
+			.insertInto('DocumentBaseline')
+			.values({
+				oid: baseline.oid,
+				snapshot: JSON.stringify(baseline.snapshot),
+				timestamp: baseline.timestamp,
+			})
+			.onConflict((cb) =>
+				cb.doUpdateSet({
+					snapshot: JSON.stringify(baseline.snapshot),
+					timestamp: baseline.timestamp,
+				}),
+			)
+			.execute();
+	};
+
+	insertAll = async (libraryId: string, baselines: DocumentBaseline[]) => {
+		const db = await this.dbs.get(libraryId);
+		await db.transaction().execute(async (tx) => {
+			for (const baseline of baselines) {
+				await this.set(libraryId, baseline, { tx });
+			}
+		});
+	};
+
+	private hydrate = (row: DocumentBaselineRow) => {
+		// avoiding extra allocation from .map by type-asserting here
+		row.snapshot = JSON.parse(row.snapshot);
+	};
+
+	applyOperations = async (
+		libraryId: string,
+		oid: string,
+		operations: StoredOperation[],
+		deletedRefs?: Ref[],
+	) => {
+		if (operations.length === 0) return;
+
+		const db = await this.dbs.get(libraryId);
+
+		await db.transaction().execute(async (tx) => {
+			let baseline = await this.get(libraryId, oid, { tx });
+			if (!baseline) {
+				baseline = {
+					oid,
+					snapshot: {},
+					timestamp: operations[0].timestamp,
+					libraryId,
+				};
+			}
+			for (const operation of operations) {
+				baseline.snapshot = applyPatch(
+					baseline.snapshot,
+					operation.data,
+					deletedRefs,
+				);
+			}
+			baseline.timestamp = operations[operations.length - 1].timestamp;
+			await this.set(libraryId, baseline, { tx });
+		});
+	};
+
+	deleteAll = async (libraryId: string) => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('DocumentBaseline').execute();
+	};
+
+	getCount = async (libraryId: string) => {
+		const db = await this.dbs.get(libraryId);
+		return (
+			(
+				await db
+					.selectFrom('DocumentBaseline')
+					.select(({ fn }) => fn.countAll<number>().as('count'))
+					.executeTakeFirst()
+			)?.count ?? 0
+		);
+	};
+}

package/src/storage/sqlShard/SqlFileMetadata.ts

@@ -0,0 +1,98 @@
+import { Kysely } from 'kysely';
+import { FileMetadataStorage } from '../Storage.js';
+import { Database, FileMetadataRow } from './tables.js';
+import { FileMetadata } from '../../types.js';
+import { FileInfo } from '../../files/FileStorage.js';
+import { Databases } from './Databases.js';
+
+export class SqlFileMetadata implements FileMetadataStorage {
+	constructor(
+		private dbs: Databases,
+		private deleteExpirationDays: number,
+		private dialect: 'postgres' | 'sqlite',
+	) {}
+
+	private attachLibrary = (libraryId: string, row: FileMetadataRow) => {
+		if (!row) return row;
+		(row as any).libraryId = libraryId;
+		return row as FileMetadata;
+	};
+
+	get = async (
+		libraryId: string,
+		fileId: string,
+	): Promise<FileMetadata | null> => {
+		const db = await this.dbs.get(libraryId);
+		const row =
+			(await db
+				.selectFrom('FileMetadata')
+				.where('fileId', '=', fileId)
+				.selectAll()
+				.executeTakeFirst()) ?? null;
+		if (row) return this.attachLibrary(libraryId, row);
+		return row;
+	};
+
+	getAll = async (libraryId: string): Promise<FileMetadata[]> => {
+		const db = await this.dbs.get(libraryId);
+		return (await db.selectFrom('FileMetadata').selectAll().execute()).map(
+			this.attachLibrary.bind(this, libraryId),
+		);
+	};
+
+	deleteAll = async (libraryId: string): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('FileMetadata').execute();
+	};
+
+	put = async (libraryId: string, fileInfo: FileInfo): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db
+			.insertInto('FileMetadata')
+			.values({
+				fileId: fileInfo.id,
+				name: fileInfo.fileName,
+				type: fileInfo.type,
+			})
+			.onConflict((cb) =>
+				cb.column('fileId').doUpdateSet({
+					name: fileInfo.fileName,
+					type: fileInfo.type,
+					pendingDeleteAt: null,
+				}),
+			)
+			.execute();
+	};
+
+	markPendingDelete = async (
+		libraryId: string,
+		fileId: string,
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db
+			.updateTable('FileMetadata')
+			.set({ pendingDeleteAt: Date.now() })
+			.where('fileId', '=', fileId)
+			.execute();
+	};
+
+	delete = async (libraryId: string, fileId: string): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('FileMetadata').where('fileId', '=', fileId).execute();
+	};
+
+	getPendingDelete = async (libraryId: string): Promise<FileMetadata[]> => {
+		const db = await this.dbs.get(libraryId);
+		return (
+			await db
+				.selectFrom('FileMetadata')
+				.where(
+					'pendingDeleteAt',
+					'<',
+					Date.now() - 1000 * 60 * 60 * 24 * this.deleteExpirationDays,
+				)
+				.selectAll()
+				.execute()
+		).map(this.attachLibrary.bind(this, libraryId));
+	};
+}

package/src/storage/sqlShard/SqlOperations.ts

@@ -0,0 +1,146 @@
+import { Operation } from '@verdant-web/common';
+import { StoredOperation } from '../../types.js';
+import { OperationStorage } from '../Storage.js';
+import { Kysely } from 'kysely';
+import { Database, OperationHistoryRow } from './tables.js';
+import { Databases } from './Databases.js';
+
+export class SqlOperations implements OperationStorage {
+	constructor(
+		private dbs: Databases,
+		private dialect: 'postgres' | 'sqlite',
+	) {}
+
+	private hydrate = (row: OperationHistoryRow) => {
+		// avoiding extra allocation from .map by type-asserting here
+		row.data = JSON.parse(row.data);
+	};
+
+	getAll = async (
+		libraryId: string,
+		oid: string,
+	): Promise<StoredOperation[]> => {
+		const db = await this.dbs.get(libraryId);
+		const raw = await db
+			.selectFrom('OperationHistory')
+			.where('oid', '=', oid)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getBeforeServerOrder = async (
+		libraryId: string,
+		beforeServerOrder: number,
+	): Promise<StoredOperation[]> => {
+		const db = await this.dbs.get(libraryId);
+		const raw = await db
+			.selectFrom('OperationHistory')
+			.where('serverOrder', '<', beforeServerOrder)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getAfterServerOrder = async (
+		libraryId: string,
+		afterServerOrder: number,
+	): Promise<StoredOperation[]> => {
+		const db = await this.dbs.get(libraryId);
+		const raw = await db
+			.selectFrom('OperationHistory')
+			.where('serverOrder', '>', afterServerOrder)
+			.orderBy('timestamp', 'asc')
+			.selectAll()
+			.execute();
+
+		raw.forEach(this.hydrate);
+		return raw as unknown as StoredOperation[];
+	};
+
+	getLatestServerOrder = async (libraryId: string): Promise<number> => {
+		const db = await this.dbs.get(libraryId);
+		return (
+			(
+				await db
+					.selectFrom('OperationHistory')
+					.orderBy('serverOrder', 'desc')
+					.select('serverOrder')
+					.limit(1)
+					.executeTakeFirst()
+			)?.serverOrder ?? 0
+		);
+	};
+
+	getCount = async (libraryId: string): Promise<number> => {
+		const db = await this.dbs.get(libraryId);
+		return (
+			(
+				await db
+					.selectFrom('OperationHistory')
+					.select(({ fn }) => fn.countAll<number>().as('count'))
+					.executeTakeFirst()
+			)?.count ?? 0
+		);
+	};
+
+	insertAll = async (
+		libraryId: string,
+		replicaId: string,
+		operations: Operation[],
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		// inserts all operations and updates server order
+		// FIXME: this whole thing is kinda sus
+		await db.transaction().execute(async (tx): Promise<void> => {
+			let orderResult = await tx
+				.selectFrom('OperationHistory')
+				.select(({ fn, val }) =>
+					fn.coalesce(fn.max<number>('serverOrder'), val(0)).as('serverOrder'),
+				)
+				.executeTakeFirst();
+			let currentServerOrder = orderResult?.serverOrder ?? 0;
+			for (const item of operations) {
+				await tx
+					.insertInto('OperationHistory')
+					.values({
+						oid: item.oid,
+						data: JSON.stringify(item.data),
+						timestamp: item.timestamp,
+						replicaId,
+						serverOrder: ++currentServerOrder,
+					})
+					.onConflict((cb) =>
+						cb.columns(['replicaId', 'oid', 'timestamp']).doNothing(),
+					)
+					.execute();
+			}
+		});
+	};
+
+	deleteAll = async (libraryId: string): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.deleteFrom('OperationHistory').execute();
+	};
+	delete = async (
+		libraryId: string,
+		operations: Operation[],
+	): Promise<void> => {
+		const db = await this.dbs.get(libraryId);
+		await db.transaction().execute(async (tx) => {
+			for (const item of operations) {
+				await tx
+					.deleteFrom('OperationHistory')
+					.where('oid', '=', item.oid)
+					.where('timestamp', '=', item.timestamp)
+					.execute();
+			}
+		});
+	};
+}