@verdant-web/store 3.12.0 → 4.0.0-next.0

package/src/persistence/PersistenceRebaser.ts

@@ -0,0 +1,171 @@
+import {
+	applyPatch,
+	assignOid,
+	isFileRef,
+	ObjectIdentifier,
+	Ref,
+} from '@verdant-web/common';
+import { Context } from '../context/context.js';
+import { AbstractTransaction, PersistenceMetadataDb } from './interfaces.js';
+
+export class PersistenceRebaser {
+	constructor(
+		private db: PersistenceMetadataDb,
+		private ctx: Pick<
+			Context,
+			'closing' | 'log' | 'time' | 'internalEvents' | 'globalEvents' | 'config'
+		>,
+	) {}
+
+	/**
+	 * Autonomous rebases are only allowed for clients who have never synced. They
+	 * keep storage clean for non-syncing clients by compressing history.
+	 */
+	tryAutonomousRebase = async () => {
+		const localReplicaInfo = await this.db.getLocalReplica();
+		if (localReplicaInfo.lastSyncedLogicalTime) return; // cannot autonomously rebase if we've synced
+		// but if we have never synced... we can rebase everything!
+		await this.runRebase(this.ctx.time.now);
+	};
+
+	/**
+	 * Attempt to autonomously rebase local documents without server intervention.
+	 * This can currently only happen for a client who has never synced before.
+	 * The goal is to allow local-only clients to compress their history to exactly
+	 * their undo stack.
+	 */
+	private runRebase = async (globalAckTimestamp: string) => {
+		if (this.ctx.closing) return;
+
+		// find all operations before the global ack
+		let lastTimestamp;
+		const toRebase = new Set<ObjectIdentifier>();
+		const transaction = this.db.transaction({
+			storeNames: ['baselines', 'operations'],
+			mode: 'readwrite',
+		});
+		let operationCount = 0;
+		await this.db.iterateAllOperations(
+			(patch) => {
+				toRebase.add(patch.oid);
+				lastTimestamp = patch.timestamp;
+				operationCount++;
+			},
+			{
+				before: globalAckTimestamp,
+				transaction,
+			},
+		);
+
+		if (!toRebase.size) {
+			return;
+		}
+
+		if (this.ctx.closing) {
+			return;
+		}
+
+		// rebase each affected document
+		let newBaselines = [];
+		for (const oid of toRebase) {
+			newBaselines.push(
+				await this.rebase(
+					oid,
+					lastTimestamp || globalAckTimestamp,
+					transaction,
+				),
+			);
+		}
+		this.ctx.globalEvents.emit('rebase');
+	};
+
+	/**
+	 * Debounces rebase attempts to avoid thrashing the database with
+	 * rebase operations.
+	 */
+	scheduleRebase = async (timestamp: string) => {
+		if (this.rebaseTimeout) {
+			clearTimeout(this.rebaseTimeout);
+		}
+		this.rebaseTimeout = setTimeout(
+			this.runRebase,
+			this.ctx.config.persistence?.rebaseTimeout ?? 10000,
+			timestamp,
+		);
+	};
+	private rebaseTimeout: NodeJS.Timeout | null = null;
+
+	rebase = async (
+		oid: ObjectIdentifier,
+		upTo: string,
+		providedTx?: AbstractTransaction,
+	) => {
+		const transaction =
+			providedTx ||
+			this.db.transaction({
+				storeNames: ['operations', 'baselines'],
+				mode: 'readwrite',
+			});
+		const baseline = await this.db.getBaseline(oid, { transaction });
+		let current: any = baseline?.snapshot || undefined;
+		let operationsApplied = 0;
+		let authz = baseline?.authz;
+		const deletedRefs: Ref[] = [];
+		await this.db.consumeEntityOperations(
+			oid,
+			(patch) => {
+				// FIXME: this seems like the wrong place to do this
+				// but it's here as a safety measure...
+				if (!baseline || patch.timestamp > baseline.timestamp) {
+					current = applyPatch(current, patch.data, deletedRefs);
+					if (patch.data.op === 'initialize') {
+						authz = patch.authz;
+					}
+				}
+				// delete all prior operations to the baseline
+				operationsApplied++;
+			},
+			{
+				to: upTo,
+				transaction,
+			},
+		);
+		if (current) {
+			assignOid(current, oid);
+		}
+		const newBaseline = {
+			oid,
+			snapshot: current,
+			timestamp: upTo,
+			authz,
+		};
+		if (newBaseline.snapshot) {
+			await this.db.setBaselines([newBaseline], { transaction });
+		} else {
+			await this.db.deleteBaseline(oid, { transaction });
+		}
+
+		this.ctx.log(
+			'debug',
+			'rebased',
+			oid,
+			'up to',
+			upTo,
+			':',
+			current,
+			'and deleted',
+			operationsApplied,
+			'operations',
+		);
+
+		// cleanup deleted refs
+		if (deletedRefs.length) {
+			const fileRefs = deletedRefs.filter(isFileRef);
+			if (fileRefs.length) {
+				this.ctx.internalEvents.emit('filesDeleted', fileRefs);
+			}
+		}
+
+		return newBaseline;
+	};
+}

package/src/{IDBService.ts → persistence/idb/IdbService.ts}

@@ -1,12 +1,13 @@
-import { Context } from './context.js';
+import { Context } from '../../context/context.js';
 import {
+	copyDatabase,
 	createAbortableTransaction,
 	isAbortError,
 	storeRequestPromise,
-} from './idb.js';
-import { Disposable } from './utils/Disposable.js';
+} from './util.js';
+import { Disposable } from '../../utils/Disposable.js';
 
-export class IDBService extends Disposable {
+export class IdbService extends Disposable {
 	protected log?: Context['log'];
 	private globalAbortController = new AbortController();
 
@@ -19,6 +20,10 @@ export class IDBService extends Disposable {
 		this.addDispose(() => {
 			this.globalAbortController.abort();
 		});
+		this.db.addEventListener('versionchange', this.onVersionChange);
+		this.addDispose(() => {
+			this.db.removeEventListener('versionchange', this.onVersionChange);
+		});
 	}
 
 	createTransaction = (
@@ -45,7 +50,7 @@ export class IDBService extends Disposable {
 		return tx;
 	};
 
-	run = async <T>(
+	run = async <T = any>(
 		storeName: string,
 		getRequest: (store: IDBObjectStore) => IDBRequest<T>,
 		opts?: {
@@ -66,7 +71,7 @@ export class IDBService extends Disposable {
 		storeName: string,
 		getRequests: (store: IDBObjectStore) => IDBRequest<T>[],
 		opts?: {
-			mode: 'readonly' | 'readwrite';
+			mode?: 'readonly' | 'readwrite';
 			transaction?: IDBTransaction;
 			abort?: AbortSignal;
 		},
@@ -80,8 +85,16 @@ export class IDBService extends Disposable {
 
 	iterate = async <T>(
 		storeName: string,
-		getRequest: (store: IDBObjectStore) => IDBRequest | IDBRequest[],
-		iterator: (value: T, store: IDBObjectStore) => boolean | void,
+		getRequest: (
+			store: IDBObjectStore,
+		) =>
+			| IDBRequest<IDBCursorWithValue | null>
+			| IDBRequest<IDBCursorWithValue | null>[],
+		iterator: (
+			value: T,
+			store: IDBObjectStore,
+			cursor: IDBCursorWithValue,
+		) => boolean | void,
 		opts?: {
 			mode?: 'readonly' | 'readwrite';
 			transaction?: IDBTransaction;
@@ -98,8 +111,12 @@ export class IDBService extends Disposable {
 						req.onsuccess = () => {
 							const cursor = req.result;
 							if (cursor) {
-								iterator(cursor.value, store);
-								cursor.continue();
+								const stop = iterator(cursor.value, store, cursor);
+								if (stop) {
+									resolve();
+								} else {
+									cursor.continue();
+								}
 							} else {
 								resolve();
 							}
@@ -119,7 +136,7 @@ export class IDBService extends Disposable {
 			request.onsuccess = () => {
 				const cursor = request.result as IDBCursorWithValue | null;
 				if (cursor) {
-					const stop = iterator(cursor.value, store);
+					const stop = iterator(cursor.value, store, cursor);
 					if (stop) {
 						resolve();
 					} else {
@@ -139,9 +156,25 @@ export class IDBService extends Disposable {
 		});
 	};
 
-	clear = (storeName: string) => {
+	clear = (storeName: string, transaction?: IDBTransaction) => {
 		return this.run<undefined>(storeName, (store) => store.clear(), {
 			mode: 'readwrite',
+			transaction,
 		});
 	};
+
+	cloneTo = async (otherDb: IDBDatabase) => {
+		await copyDatabase(this.db, otherDb);
+	};
+
+	private onVersionChange = () => {
+		this.log?.(
+			'warn',
+			`Another tab has requested a version change for ${this.db.name}`,
+		);
+		this.db.close();
+		if (typeof window !== 'undefined') {
+			window.location.reload();
+		}
+	};
 }

package/src/{files/FileStorage.ts → persistence/idb/files/IdbPersistenceFileDb.ts}

@@ -1,7 +1,12 @@
 import { FileData } from '@verdant-web/common';
-import { IDBService } from '../IDBService.js';
-import { fileToArrayBuffer } from './utils.js';
-import { getAllFromObjectStores, getSizeOfObjectStore } from '../idb.js';
+import {
+	AbstractTransaction,
+	PersistedFileData,
+	PersistenceFileDb,
+	QueryMode,
+} from '../../interfaces.js';
+import { IdbService } from '../IdbService.js';
+import { getAllFromObjectStores, getSizeOfObjectStore } from '../util.js';
 
 /**
  * When stored in IDB, replace the file blob with an array buffer
@@ -15,20 +20,27 @@ export interface StoredFileData extends Omit<FileData, 'remote' | 'file'> {
 	timestamp?: string;
 }
 
-export interface ReturnedFileData extends FileData {
-	deletedAt: number | null;
-}
+export class IdbPersistenceFileDb
+	extends IdbService
+	implements PersistenceFileDb
+{
+	transaction = (opts: {
+		mode?: QueryMode;
+		storeNames: string[];
+		abort?: AbortSignal;
+	}): AbstractTransaction => {
+		return this.createTransaction(opts.storeNames, {
+			mode: opts.mode,
+			abort: opts.abort,
+		});
+	};
 
-export class FileStorage extends IDBService {
-	addFile = async (
+	add = async (
 		file: FileData,
-		{
-			transaction,
-			downloadRemote = false,
-		}: { transaction?: IDBTransaction; downloadRemote?: boolean } = {},
-	) => {
+		options?: { transaction?: AbstractTransaction; downloadRemote?: boolean },
+	): Promise<void> => {
 		let buffer = file.file ? await fileToArrayBuffer(file.file) : undefined;
-		if (!buffer && downloadRemote && file.url) {
+		if (!buffer && options?.downloadRemote && file.url) {
 			try {
 				buffer = await fetch(file.url, {
 					method: 'GET',
@@ -41,7 +53,7 @@ export class FileStorage extends IDBService {
 				);
 			}
 		}
-		return this.run(
+		await this.run(
 			'files',
 			(store) => {
 				return store.put({
@@ -53,33 +65,25 @@ export class FileStorage extends IDBService {
 					type: file.type,
 					url: file.url,
 					buffer,
-				} as StoredFileData);
+				} satisfies StoredFileData);
+			},
+			{
+				mode: 'readwrite',
+				transaction: options?.transaction as IDBTransaction,
 			},
-			{ mode: 'readwrite', transaction },
 		);
 	};
-
-	private hydrateFileData = (raw: StoredFileData): ReturnedFileData => {
-		(raw as any).remote = raw.remote === 'true';
-		const buffer = raw.buffer;
-		delete raw.buffer;
-		(raw as unknown as FileData).file = buffer
-			? arrayBufferToBlob(buffer, raw.type)
-			: undefined;
-		return raw as unknown as ReturnedFileData;
-	};
-
 	markUploaded = async (
 		id: string,
-		{ transaction }: { transaction?: IDBTransaction } = {},
-	) => {
-		const current = await this.getFileRaw(id, { transaction });
+		options?: { transaction?: AbstractTransaction },
+	): Promise<void> => {
+		const current = await this.getFileRaw(id, options);
 
 		if (!current) {
 			throw new Error('File is not in local database');
 		}
 
-		return this.run(
+		await this.run(
 			'files',
 			(store) => {
 				return store.put({
@@ -87,62 +91,48 @@ export class FileStorage extends IDBService {
 					remote: 'true',
 				} as StoredFileData);
 			},
-			{ mode: 'readwrite', transaction },
-		);
-	};
-
-	private getFileRaw = async (
-		id: string,
-		{ transaction }: { transaction?: IDBTransaction } = {},
-	): Promise<StoredFileData | undefined> => {
-		const raw = await this.run<StoredFileData>(
-			'files',
-			(store) => {
-				return store.get(id);
+			{
+				mode: 'readwrite',
+				transaction: options?.transaction as IDBTransaction,
 			},
-			{ mode: 'readonly', transaction },
 		);
-		if (!raw) {
-			return undefined;
-		}
-		return raw;
 	};
-
-	getFile = async (
-		id: string,
-		{ transaction }: { transaction?: IDBTransaction } = {},
-	): Promise<ReturnedFileData | undefined> => {
-		const raw = await this.getFileRaw(id, { transaction });
+	get = async (
+		fileId: string,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<PersistedFileData | null> => {
+		const raw = await this.getFileRaw(fileId, options);
 		if (!raw) {
-			return undefined;
+			return null;
 		}
 		return this.hydrateFileData(raw);
 	};
-
-	deleteFile(
-		id: string,
-		{ transaction }: { transaction?: IDBTransaction } = {},
-	) {
+	delete = (
+		fileId: string,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<void> => {
 		return this.run<undefined>(
 			'files',
 			(store) => {
-				return store.delete(id);
+				return store.delete(fileId);
+			},
+			{
+				mode: 'readwrite',
+				transaction: options?.transaction as IDBTransaction,
 			},
-			{ mode: 'readwrite', transaction },
 		);
-	}
-
+	};
 	markPendingDelete = async (
-		id: string,
-		{ transaction }: { transaction?: IDBTransaction } = {},
-	) => {
-		const current = await this.getFileRaw(id, { transaction });
+		fileId: string,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<void> => {
+		const current = await this.getFileRaw(fileId, options);
 
 		if (!current) {
 			throw new Error('File is not in local database');
 		}
 
-		return this.run(
+		await this.run(
 			'files',
 			(store) => {
 				return store.put({
@@ -150,23 +140,31 @@ export class FileStorage extends IDBService {
 					deletedAt: Date.now(),
 				} as StoredFileData);
 			},
-			{ mode: 'readwrite', transaction },
+			{
+				mode: 'readwrite',
+				transaction: options?.transaction as IDBTransaction,
+			},
 		);
 	};
-
-	listUnsynced = async () => {
+	listUnsynced = async (options?: {
+		transaction?: AbstractTransaction;
+	}): Promise<PersistedFileData[]> => {
 		const raw = await this.run<StoredFileData[]>(
 			'files',
 			(store) => {
 				return store.index('remote').getAll('false');
 			},
-			{ mode: 'readonly' },
+			{ mode: 'readonly', transaction: options?.transaction as IDBTransaction },
 		);
 		return raw.map(this.hydrateFileData);
 	};
-
-	resetSyncedStatusSince = async (since: string | null) => {
-		const tx = this.createTransaction(['files'], { mode: 'readwrite' });
+	resetSyncedStatusSince = async (
+		since: string | null,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<void> => {
+		const tx: IDBTransaction =
+			(options?.transaction as any) ??
+			this.createTransaction(['files'], { mode: 'readwrite' });
 		const raw = await this.run<StoredFileData[]>(
 			'files',
 			(store) => {
@@ -194,11 +192,10 @@ export class FileStorage extends IDBService {
 			}),
 		);
 	};
-
 	iterateOverPendingDelete = (
-		iterator: (file: ReturnedFileData, store: IDBObjectStore) => void,
-		transaction?: IDBTransaction,
-	) => {
+		iterator: (file: PersistedFileData, store: IDBObjectStore) => void,
+		options?: { transaction?: IDBTransaction },
+	): Promise<void> => {
 		return this.iterate<StoredFileData>(
 			'files',
 			(store) => {
@@ -209,22 +206,67 @@ export class FileStorage extends IDBService {
 			(value, store) => {
 				iterator(this.hydrateFileData(value), store);
 			},
-			{ mode: 'readwrite', transaction },
+			{
+				mode: 'readwrite',
+				transaction: options?.transaction as IDBTransaction,
+			},
 		);
 	};
-
-	getAll = async () => {
+	getAll = async (options?: {
+		transaction?: AbstractTransaction;
+	}): Promise<PersistedFileData[]> => {
 		const [files] = await getAllFromObjectStores(this.db, ['files']);
 		return files.map(this.hydrateFileData);
 	};
-
-	stats = async () => {
+	stats = async (): Promise<{ size: { count: number; size: number } }> => {
 		return {
 			size: await getSizeOfObjectStore(this.db, 'files'),
 		};
 	};
+
+	private hydrateFileData = (raw: StoredFileData): PersistedFileData => {
+		(raw as any).remote = raw.remote === 'true';
+		const buffer = raw.buffer;
+		delete raw.buffer;
+		(raw as unknown as FileData).file = buffer
+			? arrayBufferToBlob(buffer, raw.type)
+			: undefined;
+		return raw as unknown as PersistedFileData;
+	};
+
+	private getFileRaw = async (
+		id: string,
+		{ transaction }: { transaction?: AbstractTransaction } = {},
+	): Promise<StoredFileData | undefined> => {
+		const raw = await this.run<StoredFileData>(
+			'files',
+			(store) => {
+				return store.get(id);
+			},
+			{ mode: 'readonly', transaction: transaction as IDBTransaction },
+		);
+		if (!raw) {
+			return undefined;
+		}
+		return raw;
+	};
 }
 
 export function arrayBufferToBlob(buffer: ArrayBuffer, type: string) {
 	return new Blob([buffer], { type });
 }
+
+function fileToArrayBuffer(file: File | Blob): Promise<ArrayBuffer> {
+	// special case for testing...
+	if ('__testReadBuffer' in file) {
+		return Promise.resolve<any>(file.__testReadBuffer);
+	}
+	return new Promise<ArrayBuffer>((resolve, reject) => {
+		const reader = new FileReader();
+		reader.onload = () => {
+			resolve(reader.result as ArrayBuffer);
+		};
+		reader.onerror = reject;
+		reader.readAsArrayBuffer(file);
+	});
+}