@verdant-web/server 3.0.5 → 3.1.1

package/src/storage/sqlShard/transfer.test.ts

@@ -0,0 +1,149 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { openDatabase as openUnifiedDatabase } from '../sql/database.js';
+import {
+	randomBaselines,
+	randomFileMetadata,
+	randomOperations,
+	randomReplicaInfo,
+} from './_testData/unifiedData.js';
+import { join, dirname } from 'path';
+import { transferToShards } from './transfer.js';
+import { rm, mkdir } from 'fs/promises';
+import { fileURLToPath } from 'url';
+import {
+	DocumentBaselineRow,
+	FileMetadataRow,
+	OperationHistoryRow,
+	ReplicaInfoRow,
+} from '../sql/tables.js';
+import { Kysely } from 'kysely';
+
+const testTempDir = join(
+	dirname(fileURLToPath(import.meta.url)),
+	'_testData',
+	'tmp',
+);
+
+describe('sql-shard storage transfer utility', () => {
+	const databases = new Array<Kysely<any>>();
+	beforeAll(async () => {
+		try {
+			await rm(testTempDir, { recursive: true });
+		} catch (err) {}
+		// create a temp dir
+		await mkdir(testTempDir, { recursive: true });
+	});
+	afterAll(async () => {
+		try {
+			await Promise.all(databases.map((db) => db.destroy()));
+			// empty the temp dir
+			await rm(testTempDir, { recursive: true });
+		} catch (err) {
+			// what can you do...
+			console.error(err);
+		}
+	});
+
+	it('should transfer data from unified database to shards', async () => {
+		const unifiedFile = join(testTempDir, 'unified.sqlite');
+		const { db: unifiedDb, ready } = openUnifiedDatabase(unifiedFile);
+		databases.push(unifiedDb);
+		await ready;
+		const libraryIds = ['library-1', 'library-2', 'library-3'];
+		const randomData = {} as Record<
+			string,
+			{
+				operations: OperationHistoryRow[];
+				baselines: DocumentBaselineRow[];
+				replicas: ReplicaInfoRow[];
+				fileMetadata: FileMetadataRow[];
+			}
+		>;
+		for (const libraryId of libraryIds) {
+			randomData[libraryId] = {
+				// test a lot of them - more than max sqlite var count.
+				// only doing operations here to reduce test time.
+				operations: randomOperations(libraryId, 6000),
+				baselines: randomBaselines(libraryId),
+				replicas: randomReplicaInfo(libraryId),
+				fileMetadata: randomFileMetadata(libraryId),
+			};
+
+			// have to batch these...
+			for (let i = 0; i < randomData[libraryId].operations.length; i += 1000) {
+				const toInsert = randomData[libraryId].operations.slice(i, i + 1000);
+				await unifiedDb
+					.insertInto('OperationHistory')
+					.values(toInsert)
+					.execute();
+			}
+			for (let i = 0; i < randomData[libraryId].baselines.length; i += 1000) {
+				const toInsert = randomData[libraryId].baselines.slice(i, i + 1000);
+				await unifiedDb
+					.insertInto('DocumentBaseline')
+					.values(toInsert)
+					.execute();
+			}
+			for (
+				let i = 0;
+				i < randomData[libraryId].fileMetadata.length;
+				i += 1000
+			) {
+				const toInsert = randomData[libraryId].fileMetadata.slice(i, i + 1000);
+				await unifiedDb.insertInto('FileMetadata').values(toInsert).execute();
+			}
+			for (let i = 0; i < randomData[libraryId].replicas.length; i += 1000) {
+				const toInsert = randomData[libraryId].replicas.slice(i, i + 1000);
+				await unifiedDb.insertInto('ReplicaInfo').values(toInsert).execute();
+			}
+		}
+
+		const shards = await transferToShards({
+			directory: testTempDir,
+			file: unifiedFile,
+		});
+		Object.values(shards).map((db) => databases.push(db));
+
+		expect(Object.keys(shards).sort()).toEqual(libraryIds.sort());
+
+		for (const [libraryId, db] of Object.entries(shards)) {
+			const operations = await db
+				.selectFrom('OperationHistory')
+				.selectAll()
+				.execute();
+			const baselines = await db
+				.selectFrom('DocumentBaseline')
+				.selectAll()
+				.execute();
+			const replicas = await db.selectFrom('ReplicaInfo').selectAll().execute();
+			const fileMetadata = await db
+				.selectFrom('FileMetadata')
+				.selectAll()
+				.execute();
+
+			expect(operations.length).toBe(randomData[libraryId].operations.length);
+			expect(baselines.length).toBe(randomData[libraryId].baselines.length);
+			expect(replicas.length).toBe(randomData[libraryId].replicas.length);
+			expect(fileMetadata.length).toBe(
+				randomData[libraryId].fileMetadata.length,
+			);
+
+			for (const { oid } of randomData[libraryId].operations) {
+				expect(operations.find((op) => op.oid === oid)).toBeDefined();
+			}
+			for (const { oid } of randomData[libraryId].baselines) {
+				expect(
+					baselines.find((baseline) => baseline.oid === oid),
+				).toBeDefined();
+			}
+			for (const { id } of randomData[libraryId].replicas) {
+				expect(replicas.find((replica) => replica.id === id)).toBeDefined();
+			}
+			for (const { fileId } of randomData[libraryId].fileMetadata) {
+				expect(
+					fileMetadata.find((file) => file.fileId === fileId),
+				).toBeDefined();
+			}
+		}
+	});
+});

package/src/storage/sqlShard/transfer.ts

@@ -0,0 +1,123 @@
+import { Kysely } from 'kysely';
+import { openDatabase as openUnifiedDatabase } from '../sql/database.js';
+import { Database as UnifiedDatabase } from '../sql/tables.js';
+import { Database } from './tables.js';
+import { openDatabase } from './database.js';
+
+const DEFAULT_BATCH_SIZE = 1000;
+
+export async function transferToShards({
+	file,
+	directory,
+	batchSize = DEFAULT_BATCH_SIZE,
+}: {
+	file: string;
+	directory: string;
+	batchSize?: number;
+}) {
+	console.info(
+		`Transferring from unified database (${file}) to shards (${directory})...`,
+	);
+	const { db: unifiedDb, ready } = openUnifiedDatabase(file);
+	await ready;
+	const libraries = await unifiedDb
+		.selectFrom('ReplicaInfo')
+		.select('ReplicaInfo.libraryId')
+		.distinct()
+		.execute();
+
+	const dbs = await Promise.all(
+		libraries.map(async ({ libraryId }) => {
+			const db = await openDatabase(directory, libraryId);
+			await copyLibrary(libraryId, unifiedDb, db, batchSize);
+			return db;
+		}),
+	);
+
+	return libraries.reduce(
+		(acc, { libraryId }, i) => {
+			acc[libraryId] = dbs[i];
+			return acc;
+		},
+		{} as Record<string, Kysely<Database>>,
+	);
+}
+
+async function copyLibrary(
+	libraryId: string,
+	source: Kysely<UnifiedDatabase>,
+	dest: Kysely<Database>,
+	batchSize: number,
+) {
+	const operations = await source
+		.selectFrom('OperationHistory')
+		.where('libraryId', '=', libraryId)
+		.selectAll()
+		.execute();
+	const baselines = await source
+		.selectFrom('DocumentBaseline')
+		.where('libraryId', '=', libraryId)
+		.selectAll()
+		.execute();
+	const replicas = await source
+		.selectFrom('ReplicaInfo')
+		.where('libraryId', '=', libraryId)
+		.selectAll()
+		.execute();
+	const fileMetadata = await source
+		.selectFrom('FileMetadata')
+		.where('libraryId', '=', libraryId)
+		.selectAll()
+		.execute();
+
+	for (const operation of operations) {
+		// @ts-expect-error
+		delete operation.libraryId;
+	}
+	for (const baseline of baselines) {
+		// @ts-expect-error
+		delete baseline.libraryId;
+	}
+	for (const replica of replicas) {
+		// @ts-expect-error
+		delete replica.libraryId;
+	}
+	for (const metadata of fileMetadata) {
+		// @ts-expect-error
+		delete metadata.libraryId;
+	}
+
+	await dest.transaction().execute(async (trx) => {
+		const actions: Promise<any>[] = [];
+		// batch insert data to avoid maximum vars limits
+		if (operations.length) {
+			for (let i = 0; i < operations.length; i += batchSize) {
+				const toInsert = operations.slice(i, i + batchSize);
+				actions.push(
+					trx.insertInto('OperationHistory').values(toInsert).execute(),
+				);
+			}
+		}
+		if (baselines.length) {
+			for (let i = 0; i < baselines.length; i += batchSize) {
+				const toInsert = baselines.slice(i, i + batchSize);
+				actions.push(
+					trx.insertInto('DocumentBaseline').values(toInsert).execute(),
+				);
+			}
+		}
+		if (replicas.length) {
+			for (let i = 0; i < replicas.length; i += batchSize) {
+				const toInsert = replicas.slice(i, i + batchSize);
+				actions.push(trx.insertInto('ReplicaInfo').values(toInsert).execute());
+			}
+		}
+		if (fileMetadata.length) {
+			for (let i = 0; i < fileMetadata.length; i += batchSize) {
+				const toInsert = fileMetadata.slice(i, i + batchSize);
+				actions.push(trx.insertInto('FileMetadata').values(toInsert).execute());
+			}
+		}
+		await Promise.all(actions);
+	});
+}