@verdant-web/server 3.3.10 → 4.0.0

package/src/storage/sql/migrations/v1.ts

@@ -1,64 +0,0 @@
-import { Kysely } from 'kysely';
-
-export async function up(db: Kysely<any>) {
-	await db.schema
-		.createTable('DocumentBaseline')
-		.ifNotExists()
-		.addColumn('oid', 'text')
-		.addColumn('snapshot', 'text')
-		.addColumn('timestamp', 'text', (cb) => cb.notNull())
-		.addColumn('libraryId', 'text', (cb) => cb.notNull())
-		.addPrimaryKeyConstraint('DocumentBaseline_primaryKey', [
-			'libraryId',
-			'oid',
-		])
-		.execute();
-
-	await db.schema
-		.createTable('OperationHistory')
-		.ifNotExists()
-		.addColumn('oid', 'text', (cb) => cb.notNull())
-		.addColumn('timestamp', 'text', (cb) => cb.notNull())
-		.addColumn('data', 'text', (cb) => cb.notNull())
-		.addColumn('serverOrder', 'integer', (cb) => cb.notNull().defaultTo(0))
-		.addColumn('replicaId', 'text', (cb) => cb.notNull())
-		.addColumn('libraryId', 'text', (cb) => cb.notNull())
-		.addPrimaryKeyConstraint('OperationHistory_primaryKey', [
-			'libraryId',
-			'replicaId',
-			'oid',
-			'timestamp',
-		])
-		.execute();
-
-	await db.schema
-		.createTable('ReplicaInfo')
-		.ifNotExists()
-		.addColumn('id', 'text')
-		.addColumn('libraryId', 'text', (cb) => cb.notNull())
-		.addColumn('clientId', 'text', (cb) => cb.notNull())
-		.addColumn('lastSeenWallClockTime', 'integer')
-		.addColumn('ackedLogicalTime', 'text')
-		.addColumn('type', 'integer', (cb) => cb.notNull().defaultTo(0))
-		.addColumn('ackedServerOrder', 'integer', (cb) => cb.notNull().defaultTo(0))
-		.addPrimaryKeyConstraint('ReplicaInfo_primaryKey', ['libraryId', 'id'])
-		.execute();
-
-	await db.schema
-		.createTable('FileMetadata')
-		.ifNotExists()
-		.addColumn('libraryId', 'text', (cb) => cb.notNull())
-		.addColumn('fileId', 'text', (cb) => cb.notNull())
-		.addColumn('name', 'text', (cb) => cb.notNull())
-		.addColumn('type', 'text', (cb) => cb.notNull())
-		.addColumn('pendingDeleteAt', 'integer')
-		.addPrimaryKeyConstraint('FileMetadata_primaryKey', ['libraryId', 'fileId'])
-		.execute();
-}
-
-export async function down(db: Kysely<any>) {
-	await db.schema.dropTable('DocumentBaseline').execute();
-	await db.schema.dropTable('OperationHistory').execute();
-	await db.schema.dropTable('ReplicaInfo').execute();
-	await db.schema.dropTable('FileMetadata').execute();
-}

package/src/storage/sql/migrations/v2.ts

@@ -1,20 +0,0 @@
-import { Kysely } from 'kysely';
-
-export async function up(db: Kysely<any>) {
-	// add authz column to OperationHistory and DocumentBaseline
-	await db.schema
-		.alterTable('OperationHistory')
-		.addColumn('authz', 'text', (cb) => cb.defaultTo(null))
-		.execute();
-
-	await db.schema
-		.alterTable('DocumentBaseline')
-		.addColumn('authz', 'text', (cb) => cb.defaultTo(null))
-		.execute();
-}
-
-export async function down(db: Kysely<any>) {
-	await db.schema.alterTable('OperationHistory').dropColumn('authz').execute();
-
-	await db.schema.alterTable('DocumentBaseline').dropColumn('authz').execute();
-}

package/src/storage/sql/migrations.ts

@@ -1,4 +0,0 @@
-import * as v1 from './migrations/v1.js';
-import * as v2 from './migrations/v2.js';
-
-export default { v1, v2 };

package/src/storage/sql/sqlStorage.ts

@@ -1,57 +0,0 @@
-import Database from 'better-sqlite3';
-import { Kysely, SqliteDialect } from 'kysely';
-import { StorageFactory } from '../Storage.js';
-import { SqlBaselines } from './SqlBaselines.js';
-import { SqlOperations } from './SqlOperations.js';
-import { SqlReplicas } from './SqlReplicas.js';
-import { Database as DatabaseTypes } from './tables.js';
-import { SqlFileMetadata } from './SqlFileMetadata.js';
-import { migrateToLatest } from '@a-type/kysely';
-import migrations from './migrations.js';
-import { openDatabase } from './database.js';
-
-export const sqlStorage = ({
-	databaseFile: dbFile,
-	skipMigrations,
-	disableWal,
-}: {
-	databaseFile: string;
-	skipMigrations?: boolean;
-	disableWal?: boolean;
-}): StorageFactory => {
-	const { db, ready } = openDatabase(dbFile, {
-		skipMigrations,
-		disableWal,
-	});
-	return (options) => {
-		const baselines = new SqlBaselines(db, 'sqlite');
-		const operations = new SqlOperations(db, 'sqlite');
-		const replicas = new SqlReplicas(
-			db,
-			options.replicaTruancyMinutes,
-			'sqlite',
-		);
-		const fileMetadata = new SqlFileMetadata(
-			db,
-			options.fileDeleteExpirationDays,
-			'sqlite',
-		);
-		const close = async () => {
-			storage.open = false;
-			await db.destroy();
-		};
-		const storage = {
-			baselines,
-			operations,
-			replicas,
-			fileMetadata,
-			close,
-			open: false,
-			ready,
-		};
-		ready.then(() => {
-			storage.open = true;
-		});
-		return storage;
-	};
-};

package/src/storage/sql/tables.ts

@@ -1,51 +0,0 @@
-import { ReplicaType } from '@verdant-web/common';
-import { Generated, Selectable } from 'kysely';
-
-export interface Database {
-	OperationHistory: OperationHistoryTable;
-	DocumentBaseline: DocumentBaselineTable;
-	ReplicaInfo: ReplicaInfoTable;
-	FileMetadata: FileMetadataTable;
-}
-
-export interface OperationHistoryTable {
-	oid: string;
-	timestamp: string;
-	data: string;
-	serverOrder: number;
-	replicaId: string;
-	libraryId: string;
-	authz: string | null;
-}
-export type OperationHistoryRow = Selectable<OperationHistoryTable>;
-
-export interface DocumentBaselineTable {
-	oid: string;
-	snapshot: string;
-	timestamp: string;
-	libraryId: string;
-	authz: string | null;
-}
-export type DocumentBaselineRow = Selectable<DocumentBaselineTable>;
-
-export interface ReplicaInfoTable {
-	id: string;
-	libraryId: string;
-	clientId: string;
-	lastSeenWallClockTime: number | null;
-	ackedLogicalTime: string | null;
-	type: ReplicaType;
-	ackedServerOrder: Generated<number>;
-}
-
-export type ReplicaInfoRow = Selectable<ReplicaInfoTable>;
-
-export interface FileMetadataTable {
-	libraryId: string;
-	fileId: string;
-	name: string;
-	type: string;
-	pendingDeleteAt: number | null;
-}
-
-export type FileMetadataRow = Selectable<FileMetadataTable>;

package/src/storage/sqlShard/transfer.test.ts

@@ -1,149 +0,0 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { openDatabase as openUnifiedDatabase } from '../sql/database.js';
-import {
-	randomBaselines,
-	randomFileMetadata,
-	randomOperations,
-	randomReplicaInfo,
-} from './_testData/unifiedData.js';
-import { join, dirname } from 'path';
-import { transferToShards } from './transfer.js';
-import { rm, mkdir } from 'fs/promises';
-import { fileURLToPath } from 'url';
-import {
-	DocumentBaselineRow,
-	FileMetadataRow,
-	OperationHistoryRow,
-	ReplicaInfoRow,
-} from '../sql/tables.js';
-import { Kysely } from 'kysely';
-
-const testTempDir = join(
-	dirname(fileURLToPath(import.meta.url)),
-	'_testData',
-	'tmp',
-);
-
-describe('sql-shard storage transfer utility', () => {
-	const databases = new Array<Kysely<any>>();
-	beforeAll(async () => {
-		try {
-			await rm(testTempDir, { recursive: true });
-		} catch (err) {}
-		// create a temp dir
-		await mkdir(testTempDir, { recursive: true });
-	});
-	afterAll(async () => {
-		try {
-			await Promise.all(databases.map((db) => db.destroy()));
-			// empty the temp dir
-			await rm(testTempDir, { recursive: true });
-		} catch (err) {
-			// what can you do...
-			console.error(err);
-		}
-	});
-
-	it('should transfer data from unified database to shards', async () => {
-		const unifiedFile = join(testTempDir, 'unified.sqlite');
-		const { db: unifiedDb, ready } = openUnifiedDatabase(unifiedFile);
-		databases.push(unifiedDb);
-		await ready;
-		const libraryIds = ['library-1', 'library-2', 'library-3'];
-		const randomData = {} as Record<
-			string,
-			{
-				operations: OperationHistoryRow[];
-				baselines: DocumentBaselineRow[];
-				replicas: ReplicaInfoRow[];
-				fileMetadata: FileMetadataRow[];
-			}
-		>;
-		for (const libraryId of libraryIds) {
-			randomData[libraryId] = {
-				// test a lot of them - more than max sqlite var count.
-				// only doing operations here to reduce test time.
-				operations: randomOperations(libraryId, 6000),
-				baselines: randomBaselines(libraryId),
-				replicas: randomReplicaInfo(libraryId),
-				fileMetadata: randomFileMetadata(libraryId),
-			};
-
-			// have to batch these...
-			for (let i = 0; i < randomData[libraryId].operations.length; i += 1000) {
-				const toInsert = randomData[libraryId].operations.slice(i, i + 1000);
-				await unifiedDb
-					.insertInto('OperationHistory')
-					.values(toInsert)
-					.execute();
-			}
-			for (let i = 0; i < randomData[libraryId].baselines.length; i += 1000) {
-				const toInsert = randomData[libraryId].baselines.slice(i, i + 1000);
-				await unifiedDb
-					.insertInto('DocumentBaseline')
-					.values(toInsert)
-					.execute();
-			}
-			for (
-				let i = 0;
-				i < randomData[libraryId].fileMetadata.length;
-				i += 1000
-			) {
-				const toInsert = randomData[libraryId].fileMetadata.slice(i, i + 1000);
-				await unifiedDb.insertInto('FileMetadata').values(toInsert).execute();
-			}
-			for (let i = 0; i < randomData[libraryId].replicas.length; i += 1000) {
-				const toInsert = randomData[libraryId].replicas.slice(i, i + 1000);
-				await unifiedDb.insertInto('ReplicaInfo').values(toInsert).execute();
-			}
-		}
-
-		const shards = await transferToShards({
-			directory: testTempDir,
-			file: unifiedFile,
-		});
-		Object.values(shards).map((db) => databases.push(db));
-
-		expect(Object.keys(shards).sort()).toEqual(libraryIds.sort());
-
-		for (const [libraryId, db] of Object.entries(shards)) {
-			const operations = await db
-				.selectFrom('OperationHistory')
-				.selectAll()
-				.execute();
-			const baselines = await db
-				.selectFrom('DocumentBaseline')
-				.selectAll()
-				.execute();
-			const replicas = await db.selectFrom('ReplicaInfo').selectAll().execute();
-			const fileMetadata = await db
-				.selectFrom('FileMetadata')
-				.selectAll()
-				.execute();
-
-			expect(operations.length).toBe(randomData[libraryId].operations.length);
-			expect(baselines.length).toBe(randomData[libraryId].baselines.length);
-			expect(replicas.length).toBe(randomData[libraryId].replicas.length);
-			expect(fileMetadata.length).toBe(
-				randomData[libraryId].fileMetadata.length,
-			);
-
-			for (const { oid } of randomData[libraryId].operations) {
-				expect(operations.find((op) => op.oid === oid)).toBeDefined();
-			}
-			for (const { oid } of randomData[libraryId].baselines) {
-				expect(
-					baselines.find((baseline) => baseline.oid === oid),
-				).toBeDefined();
-			}
-			for (const { id } of randomData[libraryId].replicas) {
-				expect(replicas.find((replica) => replica.id === id)).toBeDefined();
-			}
-			for (const { fileId } of randomData[libraryId].fileMetadata) {
-				expect(
-					fileMetadata.find((file) => file.fileId === fileId),
-				).toBeDefined();
-			}
-		}
-	});
-});

package/src/storage/sqlShard/transfer.ts

@@ -1,123 +0,0 @@
-import { Kysely } from 'kysely';
-import { openDatabase as openUnifiedDatabase } from '../sql/database.js';
-import { Database as UnifiedDatabase } from '../sql/tables.js';
-import { Database } from './tables.js';
-import { openDatabase } from './database.js';
-
-const DEFAULT_BATCH_SIZE = 1000;
-
-export async function transferToShards({
-	file,
-	directory,
-	batchSize = DEFAULT_BATCH_SIZE,
-}: {
-	file: string;
-	directory: string;
-	batchSize?: number;
-}) {
-	console.info(
-		`Transferring from unified database (${file}) to shards (${directory})...`,
-	);
-	const { db: unifiedDb, ready } = openUnifiedDatabase(file);
-	await ready;
-	const libraries = await unifiedDb
-		.selectFrom('ReplicaInfo')
-		.select('ReplicaInfo.libraryId')
-		.distinct()
-		.execute();
-
-	const dbs = await Promise.all(
-		libraries.map(async ({ libraryId }) => {
-			const db = await openDatabase(directory, libraryId);
-			await copyLibrary(libraryId, unifiedDb, db, batchSize);
-			return db;
-		}),
-	);
-
-	return libraries.reduce(
-		(acc, { libraryId }, i) => {
-			acc[libraryId] = dbs[i];
-			return acc;
-		},
-		{} as Record<string, Kysely<Database>>,
-	);
-}
-
-async function copyLibrary(
-	libraryId: string,
-	source: Kysely<UnifiedDatabase>,
-	dest: Kysely<Database>,
-	batchSize: number,
-) {
-	const operations = await source
-		.selectFrom('OperationHistory')
-		.where('libraryId', '=', libraryId)
-		.selectAll()
-		.execute();
-	const baselines = await source
-		.selectFrom('DocumentBaseline')
-		.where('libraryId', '=', libraryId)
-		.selectAll()
-		.execute();
-	const replicas = await source
-		.selectFrom('ReplicaInfo')
-		.where('libraryId', '=', libraryId)
-		.selectAll()
-		.execute();
-	const fileMetadata = await source
-		.selectFrom('FileMetadata')
-		.where('libraryId', '=', libraryId)
-		.selectAll()
-		.execute();
-
-	for (const operation of operations) {
-		// @ts-expect-error
-		delete operation.libraryId;
-	}
-	for (const baseline of baselines) {
-		// @ts-expect-error
-		delete baseline.libraryId;
-	}
-	for (const replica of replicas) {
-		// @ts-expect-error
-		delete replica.libraryId;
-	}
-	for (const metadata of fileMetadata) {
-		// @ts-expect-error
-		delete metadata.libraryId;
-	}
-
-	await dest.transaction().execute(async (trx) => {
-		const actions: Promise<any>[] = [];
-		// batch insert data to avoid maximum vars limits
-		if (operations.length) {
-			for (let i = 0; i < operations.length; i += batchSize) {
-				const toInsert = operations.slice(i, i + batchSize);
-				actions.push(
-					trx.insertInto('OperationHistory').values(toInsert).execute(),
-				);
-			}
-		}
-		if (baselines.length) {
-			for (let i = 0; i < baselines.length; i += batchSize) {
-				const toInsert = baselines.slice(i, i + batchSize);
-				actions.push(
-					trx.insertInto('DocumentBaseline').values(toInsert).execute(),
-				);
-			}
-		}
-		if (replicas.length) {
-			for (let i = 0; i < replicas.length; i += batchSize) {
-				const toInsert = replicas.slice(i, i + batchSize);
-				actions.push(trx.insertInto('ReplicaInfo').values(toInsert).execute());
-			}
-		}
-		if (fileMetadata.length) {
-			for (let i = 0; i < fileMetadata.length; i += batchSize) {
-				const toInsert = fileMetadata.slice(i, i + batchSize);
-				actions.push(trx.insertInto('FileMetadata').values(toInsert).execute());
-			}
-		}
-		await Promise.all(actions);
-	});
-}