@verdant-web/server 3.3.10 → 4.0.0

package/dist/esm/storage/sqlShard/transfer.test.js

@@ -1,109 +0,0 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { openDatabase as openUnifiedDatabase } from '../sql/database.js';
-import { randomBaselines, randomFileMetadata, randomOperations, randomReplicaInfo, } from './_testData/unifiedData.js';
-import { join, dirname } from 'path';
-import { transferToShards } from './transfer.js';
-import { rm, mkdir } from 'fs/promises';
-import { fileURLToPath } from 'url';
-const testTempDir = join(dirname(fileURLToPath(import.meta.url)), '_testData', 'tmp');
-describe('sql-shard storage transfer utility', () => {
-    const databases = new Array();
-    beforeAll(async () => {
-        try {
-            await rm(testTempDir, { recursive: true });
-        }
-        catch (err) { }
-        // create a temp dir
-        await mkdir(testTempDir, { recursive: true });
-    });
-    afterAll(async () => {
-        try {
-            await Promise.all(databases.map((db) => db.destroy()));
-            // empty the temp dir
-            await rm(testTempDir, { recursive: true });
-        }
-        catch (err) {
-            // what can you do...
-            console.error(err);
-        }
-    });
-    it('should transfer data from unified database to shards', async () => {
-        const unifiedFile = join(testTempDir, 'unified.sqlite');
-        const { db: unifiedDb, ready } = openUnifiedDatabase(unifiedFile);
-        databases.push(unifiedDb);
-        await ready;
-        const libraryIds = ['library-1', 'library-2', 'library-3'];
-        const randomData = {};
-        for (const libraryId of libraryIds) {
-            randomData[libraryId] = {
-                // test a lot of them - more than max sqlite var count.
-                // only doing operations here to reduce test time.
-                operations: randomOperations(libraryId, 6000),
-                baselines: randomBaselines(libraryId),
-                replicas: randomReplicaInfo(libraryId),
-                fileMetadata: randomFileMetadata(libraryId),
-            };
-            // have to batch these...
-            for (let i = 0; i < randomData[libraryId].operations.length; i += 1000) {
-                const toInsert = randomData[libraryId].operations.slice(i, i + 1000);
-                await unifiedDb
-                    .insertInto('OperationHistory')
-                    .values(toInsert)
-                    .execute();
-            }
-            for (let i = 0; i < randomData[libraryId].baselines.length; i += 1000) {
-                const toInsert = randomData[libraryId].baselines.slice(i, i + 1000);
-                await unifiedDb
-                    .insertInto('DocumentBaseline')
-                    .values(toInsert)
-                    .execute();
-            }
-            for (let i = 0; i < randomData[libraryId].fileMetadata.length; i += 1000) {
-                const toInsert = randomData[libraryId].fileMetadata.slice(i, i + 1000);
-                await unifiedDb.insertInto('FileMetadata').values(toInsert).execute();
-            }
-            for (let i = 0; i < randomData[libraryId].replicas.length; i += 1000) {
-                const toInsert = randomData[libraryId].replicas.slice(i, i + 1000);
-                await unifiedDb.insertInto('ReplicaInfo').values(toInsert).execute();
-            }
-        }
-        const shards = await transferToShards({
-            directory: testTempDir,
-            file: unifiedFile,
-        });
-        Object.values(shards).map((db) => databases.push(db));
-        expect(Object.keys(shards).sort()).toEqual(libraryIds.sort());
-        for (const [libraryId, db] of Object.entries(shards)) {
-            const operations = await db
-                .selectFrom('OperationHistory')
-                .selectAll()
-                .execute();
-            const baselines = await db
-                .selectFrom('DocumentBaseline')
-                .selectAll()
-                .execute();
-            const replicas = await db.selectFrom('ReplicaInfo').selectAll().execute();
-            const fileMetadata = await db
-                .selectFrom('FileMetadata')
-                .selectAll()
-                .execute();
-            expect(operations.length).toBe(randomData[libraryId].operations.length);
-            expect(baselines.length).toBe(randomData[libraryId].baselines.length);
-            expect(replicas.length).toBe(randomData[libraryId].replicas.length);
-            expect(fileMetadata.length).toBe(randomData[libraryId].fileMetadata.length);
-            for (const { oid } of randomData[libraryId].operations) {
-                expect(operations.find((op) => op.oid === oid)).toBeDefined();
-            }
-            for (const { oid } of randomData[libraryId].baselines) {
-                expect(baselines.find((baseline) => baseline.oid === oid)).toBeDefined();
-            }
-            for (const { id } of randomData[libraryId].replicas) {
-                expect(replicas.find((replica) => replica.id === id)).toBeDefined();
-            }
-            for (const { fileId } of randomData[libraryId].fileMetadata) {
-                expect(fileMetadata.find((file) => file.fileId === fileId)).toBeDefined();
-            }
-        }
-    });
-});
-//# sourceMappingURL=transfer.test.js.map

package/dist/esm/storage/sqlShard/transfer.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"transfer.test.js","sourceRoot":"","sources":["../../../../src/storage/sqlShard/transfer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACnE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EACN,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GACjB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AASpC,MAAM,WAAW,GAAG,IAAI,CACvB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACvC,WAAW,EACX,KAAK,CACL,CAAC;AAEF,QAAQ,CAAC,oCAAoC,EAAE,GAAG,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,KAAK,EAAe,CAAC;IAC3C,SAAS,CAAC,KAAK,IAAI,EAAE;QACpB,IAAI,CAAC;YACJ,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAA,CAAC;QAChB,oBAAoB;QACpB,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,KAAK,IAAI,EAAE;QACnB,IAAI,CAAC;YACJ,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACvD,qBAAqB;YACrB,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,qBAAqB;YACrB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;QACxD,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAClE,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1B,MAAM,KAAK,CAAC;QACZ,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,EAQlB,CAAC;QACF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,UAAU,CAAC,SAAS,CAAC,GAAG;gBACvB,uDAAuD;gBACvD,kDAAkD;gBAClD,UAAU,EAAE,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC;gBAC7C,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC;gBACrC,QAAQ,EAAE,iBAAiB,CAAC,SAAS,CAAC;gBACtC,YAAY,EAAE,kBAAkB,CAAC,SAAS,CAAC;aAC3C,CAAC;YAEF,yBAAyB;YACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACxE,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACrE,MAAM,SAAS;qBACb,UAAU,CAAC,kBAAkB,CAAC;qBAC9B,MAAM,CAAC,QAAQ,CAAC;qBAChB,OAAO,EAAE,CAAC;YACb,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACvE,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACpE,MAAM,SAAS;qBACb,UAAU,CAAC,kBAAkB,CAAC;qBAC9B,MAAM,CAAC,QAAQ,CAAC;qBAChB,OAAO,EAAE,CAAC;YACb,CAAC;YACD,KACC,IAAI,CAAC,GAAG,CAAC,EACT,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,MAAM,EAC7C,CAAC,IAAI,IAAI,EACR,CAAC;gBACF,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACvE,MAAM,SAAS,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;YACvE,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtE,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACnE,MAAM,SAAS,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;YACtE,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC;YACrC,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,WAAW;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAE9D,KAAK,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,MAAM,UAAU,GAAG,MAAM,EAAE;iBACzB,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YACZ,MAAM,SAAS,GAAG,MAAM,EAAE;iBACxB,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,CAAC;YAC1E,MAAM,YAAY,GAAG,MAAM,EAAE;iBAC3B,UAAU,CAAC,cAAc,CAAC;iBAC1B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YAEZ,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACxE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACtE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,CAC/B,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,MAAM,CACzC,CAAC;YAEF,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE,CAAC;gBACxD,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/D,CAAC;YACD,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,EAAE,CAAC;gBACvD,MAAM,CACL,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,KAAK,GAAG,CAAC,CAClD,CAAC,WAAW,EAAE,CAAC;YACjB,CAAC;YACD,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACrD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACrE,CAAC;YACD,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,EAAE,CAAC;gBAC7D,MAAM,CACL,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,CACnD,CAAC,WAAW,EAAE,CAAC;YACjB,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}

package/src/MessageSender.ts

@@ -1,10 +0,0 @@
-import { ServerMessage } from '@verdant-web/common';
-
-export interface MessageSender {
-	broadcast(
-		libraryId: string,
-		message: ServerMessage,
-		omitKeys?: string[],
-	): void;
-	send(libraryId: string, clientKey: string, message: ServerMessage): void;
-}

package/src/Presence.ts

@@ -1,86 +0,0 @@
-import {
-	initialInternalPresence,
-	UserInfo,
-	UserInfoUpdate,
-} from '@verdant-web/common';
-import EventEmitter from 'events';
-import { UserProfiles } from './Profiles.js';
-
-/**
- * Stores client presence in-memory for connected
- * clients
- */
-export class Presence extends EventEmitter {
-	private presences: Map<
-		string,
-		{
-			presences: Record<string, UserInfo<any, any>>;
-			replicaToUser: Record<string, string>;
-			userToReplica: Record<string, Set<string>>;
-		}
-	> = new Map();
-
-	constructor(private profiles: UserProfiles<any>) {
-		super();
-	}
-
-	private getLibrary = (libraryId: string) => {
-		let map = this.presences.get(libraryId);
-		if (!map) {
-			map = {
-				presences: {},
-				replicaToUser: {},
-				userToReplica: {},
-			};
-			this.presences.set(libraryId, map);
-		}
-		return map;
-	};
-
-	set = async (libraryId: string, userId: string, userInfo: UserInfoUpdate) => {
-		const lib = this.getLibrary(libraryId);
-		if (!lib.presences[userId]) {
-			lib.presences[userId] = {
-				...userInfo,
-				profile: await this.profiles.get(userId),
-				internal: userInfo.internal || initialInternalPresence,
-			};
-		} else {
-			Object.assign(lib.presences[userId], userInfo);
-		}
-		lib.replicaToUser[userInfo.replicaId] = userId;
-		lib.userToReplica[userId] = lib.userToReplica[userId] || new Set<string>();
-		lib.userToReplica[userId].add(userInfo.replicaId);
-
-		return lib.presences[userId]!;
-	};
-
-	get = (libraryId: string, userId: string) => {
-		return this.presences.get(libraryId)?.presences[userId];
-	};
-
-	removeReplica = (libraryId: string, replicaId: string) => {
-		const lib = this.getLibrary(libraryId);
-		const userId = lib.replicaToUser[replicaId];
-		if (!userId) return;
-
-		lib.userToReplica[userId].delete(replicaId);
-		if (lib.userToReplica[userId].size === 0) {
-			delete lib.presences[userId];
-			this.emit('lost', libraryId, replicaId, userId);
-
-			// memory cleanup
-			if (Object.keys(lib.presences).length === 0) {
-				this.clear(libraryId);
-			}
-		}
-	};
-
-	all = (libraryId: string) => {
-		return this.presences.get(libraryId)?.presences || {};
-	};
-
-	clear = (libraryId: string) => {
-		this.presences.delete(libraryId);
-	};
-}

package/src/ReplicaKeepaliveTimers.ts

@@ -1,34 +0,0 @@
-import { EventSubscriber } from '@verdant-web/common';
-
-/**
- * A set of timers per-replica which are
- * refreshed whenever the replica sends
- * a message to the server. If a timer expires,
- * the replica is removed from presence. This
- * is a failsafe for socket-based replicas but
- * essential for pull-based replicas.
- */
-export class ReplicaKeepaliveTimers extends EventSubscriber<{
-	lost: (libraryId: string, replicaId: string) => void;
-}> {
-	private timers = new Map<string, NodeJS.Timeout>();
-
-	constructor(private timeout = 30 * 1000) {
-		super();
-	}
-
-	refresh = (libraryId: string, replicaId: string) => {
-		const existing = this.timers.get(replicaId);
-		if (existing) {
-			clearTimeout(existing);
-		}
-
-		this.timers.set(
-			replicaId,
-			setTimeout(() => {
-				this.emit('lost', libraryId, replicaId);
-				this.timers.delete(replicaId);
-			}, this.timeout),
-		);
-	};
-}