@verdant-web/store 4.0.0 → 4.1.0-alpha.1

package/src/persistence/PersistenceRebaser.ts

@@ -7,13 +7,22 @@ import {
 } from '@verdant-web/common';
 import { Context } from '../context/context.js';
 import { AbstractTransaction, PersistenceMetadataDb } from './interfaces.js';
+import type { PersistenceMetadata } from './PersistenceMetadata.js';
 
 export class PersistenceRebaser {
 	constructor(
 		private db: PersistenceMetadataDb,
+		private meta: PersistenceMetadata,
 		private ctx: Pick<
 			Context,
-			'closing' | 'log' | 'time' | 'internalEvents' | 'globalEvents' | 'config'
+			| 'closing'
+			| 'log'
+			| 'time'
+			| 'internalEvents'
+			| 'globalEvents'
+			| 'config'
+			| 'closeLock'
+			| 'persistenceShutdownHandler'
 		>,
 	) {}
 
@@ -22,9 +31,12 @@ export class PersistenceRebaser {
 	 * keep storage clean for non-syncing clients by compressing history.
 	 */
 	tryAutonomousRebase = async () => {
-		const localReplicaInfo = await this.db.getLocalReplica();
+		const localReplicaInfo = await this.meta.getLocalReplica();
 		if (localReplicaInfo.lastSyncedLogicalTime) return; // cannot autonomously rebase if we've synced
+		if (this.ctx.closing || this.ctx.persistenceShutdownHandler.isShuttingDown)
+			return;
 		// but if we have never synced... we can rebase everything!
+		this.ctx.log('debug', 'Running autonomous library rebase');
 		await this.runRebase(this.ctx.time.now);
 	};
 
@@ -35,47 +47,55 @@ export class PersistenceRebaser {
 	 * their undo stack.
 	 */
 	private runRebase = async (globalAckTimestamp: string) => {
-		if (this.ctx.closing) return;
-
-		// find all operations before the global ack
-		let lastTimestamp;
-		const toRebase = new Set<ObjectIdentifier>();
-		const transaction = this.db.transaction({
-			storeNames: ['baselines', 'operations'],
-			mode: 'readwrite',
-		});
-		let operationCount = 0;
-		await this.db.iterateAllOperations(
-			(patch) => {
-				toRebase.add(patch.oid);
-				lastTimestamp = patch.timestamp;
-				operationCount++;
-			},
+		if (this.ctx.closing || this.ctx.persistenceShutdownHandler.isShuttingDown)
+			return;
+
+		await this.db.transaction(
 			{
-				before: globalAckTimestamp,
-				transaction,
+				storeNames: ['baselines', 'operations'],
+				mode: 'readwrite',
 			},
-		);
+			async (transaction) => {
+				// find all operations before the global ack
+				const toRebase = new Set<ObjectIdentifier>();
+				let lastTimestamp;
+				let operationCount = 0;
+				await this.db.iterateAllOperations(
+					(patch) => {
+						toRebase.add(patch.oid);
+						lastTimestamp = patch.timestamp;
+						operationCount++;
+					},
+					{
+						before: globalAckTimestamp,
+						transaction,
+					},
+				);
 
-		if (!toRebase.size) {
-			return;
-		}
+				if (!toRebase.size) {
+					return;
+				}
 
-		if (this.ctx.closing) {
-			return;
-		}
+				if (
+					this.ctx.closing ||
+					this.ctx.persistenceShutdownHandler.isShuttingDown
+				) {
+					return;
+				}
 
-		// rebase each affected document
-		let newBaselines = [];
-		for (const oid of toRebase) {
-			newBaselines.push(
-				await this.rebase(
-					oid,
-					lastTimestamp || globalAckTimestamp,
-					transaction,
-				),
-			);
-		}
+				// rebase each affected document
+				let newBaselines = [];
+				for (const oid of toRebase) {
+					newBaselines.push(
+						await this.rebase(
+							oid,
+							lastTimestamp || globalAckTimestamp,
+							transaction,
+						),
+					);
+				}
+			},
+		);
 		this.ctx.globalEvents.emit('rebase');
 	};
 
@@ -92,26 +112,28 @@ export class PersistenceRebaser {
 			this.ctx.config.persistence?.rebaseTimeout ?? 10000,
 			timestamp,
 		);
+		this.ctx.log('debug', 'Scheduled rebase up to global ack', timestamp);
 	};
 	private rebaseTimeout: NodeJS.Timeout | null = null;
 
-	rebase = async (
+	private rebase = async (
 		oid: ObjectIdentifier,
 		upTo: string,
-		providedTx?: AbstractTransaction,
+		transaction: AbstractTransaction,
 	) => {
-		const transaction =
-			providedTx ||
-			this.db.transaction({
-				storeNames: ['operations', 'baselines'],
-				mode: 'readwrite',
-			});
+		if (this.ctx.closing || this.ctx.persistenceShutdownHandler.isShuttingDown)
+			return;
+
 		const baseline = await this.db.getBaseline(oid, { transaction });
 		let current: any = baseline?.snapshot || undefined;
 		let operationsApplied = 0;
 		let authz = baseline?.authz;
 		const deletedRefs: Ref[] = [];
-		await this.db.consumeEntityOperations(
+
+		if (this.ctx.closing || this.ctx.persistenceShutdownHandler.isShuttingDown)
+			return;
+
+		await this.db.iterateEntityOperations(
 			oid,
 			(patch) => {
 				// FIXME: this seems like the wrong place to do this
@@ -139,32 +161,45 @@ export class PersistenceRebaser {
 			timestamp: upTo,
 			authz,
 		};
-		if (newBaseline.snapshot) {
-			await this.db.setBaselines([newBaseline], { transaction });
-		} else {
-			await this.db.deleteBaseline(oid, { transaction });
-		}
 
-		this.ctx.log(
-			'debug',
-			'rebased',
-			oid,
-			'up to',
-			upTo,
-			':',
-			current,
-			'and deleted',
-			operationsApplied,
-			'operations',
-		);
+		// still time to cancel now...
+		if (this.ctx.closing || this.ctx.persistenceShutdownHandler.isShuttingDown)
+			return;
 
-		// cleanup deleted refs
-		if (deletedRefs.length) {
-			const fileRefs = deletedRefs.filter(isFileRef);
-			if (fileRefs.length) {
-				this.ctx.internalEvents.emit('filesDeleted', fileRefs);
+		// FROM HERE, WE ARE COMMITTED TO THE REBASE -- otherwise data will be corrupted.
+		this.ctx.closeLock = (async () => {
+			if (newBaseline.snapshot) {
+				await this.db.setBaselines([newBaseline], { transaction });
+			} else {
+				await this.db.deleteBaseline(oid, { transaction });
 			}
-		}
+
+			await this.db.deleteEntityOperations(oid, {
+				to: upTo,
+				transaction,
+			});
+
+			this.ctx.log(
+				'debug',
+				'rebased',
+				oid,
+				'up to',
+				upTo,
+				':',
+				current,
+				'and deleted',
+				operationsApplied,
+				'operations',
+			);
+
+			// cleanup deleted refs
+			if (deletedRefs.length) {
+				const fileRefs = deletedRefs.filter(isFileRef);
+				if (fileRefs.length) {
+					this.ctx.internalEvents.emit('filesDeleted', fileRefs);
+				}
+			}
+		})();
 
 		return newBaseline;
 	};

package/src/persistence/idb/IdbService.ts

@@ -1,6 +1,5 @@
 import { Context } from '../../context/context.js';
 import {
-	copyDatabase,
 	createAbortableTransaction,
 	isAbortError,
 	storeRequestPromise,
@@ -33,21 +32,40 @@ export class IdbService extends Disposable {
 			abort?: AbortSignal;
 		},
 	) => {
-		const tx = createAbortableTransaction(
-			this.db,
-			storeNames,
-			opts?.mode || 'readonly',
-			opts?.abort,
-			this.log,
-		);
-		this.globalAbortController.signal.addEventListener('abort', tx.abort);
-		tx.addEventListener('complete', () => {
-			this.globalAbortController.signal.removeEventListener('abort', tx.abort);
-		});
-		tx.addEventListener('error', () => {
-			this.globalAbortController.signal.removeEventListener('abort', tx.abort);
-		});
-		return tx;
+		try {
+			if (this.globalAbortController.signal.aborted) {
+				throw new Error('Global abort signal is already aborted');
+			}
+			const tx = createAbortableTransaction(
+				this.db,
+				storeNames,
+				opts?.mode || 'readonly',
+				opts?.abort,
+				this.log,
+			);
+			this.globalAbortController.signal.addEventListener('abort', tx.abort);
+			tx.addEventListener('complete', () => {
+				this.globalAbortController.signal.removeEventListener(
+					'abort',
+					tx.abort,
+				);
+			});
+			tx.addEventListener('error', () => {
+				this.globalAbortController.signal.removeEventListener(
+					'abort',
+					tx.abort,
+				);
+			});
+			return tx;
+		} catch (err) {
+			this.log?.(
+				'error',
+				'Failed to create abortable transaction for store names',
+				storeNames,
+				err,
+			);
+			throw err;
+		}
 	};
 
 	run = async <T = any>(
@@ -163,18 +181,18 @@ export class IdbService extends Disposable {
 		});
 	};
 
-	cloneTo = async (otherDb: IDBDatabase) => {
-		await copyDatabase(this.db, otherDb);
-	};
-
-	private onVersionChange = () => {
+	private onVersionChange = (ev: IDBVersionChangeEvent) => {
 		this.log?.(
 			'warn',
 			`Another tab has requested a version change for ${this.db.name}`,
 		);
 		this.db.close();
 		if (typeof window !== 'undefined') {
-			window.location.reload();
+			try {
+				window.location.reload();
+			} catch (err) {
+				this.log?.('error', 'Failed to reload the page', err);
+			}
 		}
 	};
 }

package/src/persistence/idb/files/IdbPersistenceFileDb.ts

@@ -3,10 +3,10 @@ import {
 	AbstractTransaction,
 	PersistedFileData,
 	PersistenceFileDb,
-	QueryMode,
 } from '../../interfaces.js';
 import { IdbService } from '../IdbService.js';
 import { getAllFromObjectStores, getSizeOfObjectStore } from '../util.js';
+import { Context } from '../../../internal.js';
 
 /**
  * When stored in IDB, replace the file blob with an array buffer
@@ -24,35 +24,11 @@ export class IdbPersistenceFileDb
 	extends IdbService
 	implements PersistenceFileDb
 {
-	transaction = (opts: {
-		mode?: QueryMode;
-		storeNames: string[];
-		abort?: AbortSignal;
-	}): AbstractTransaction => {
-		return this.createTransaction(opts.storeNames, {
-			mode: opts.mode,
-			abort: opts.abort,
-		});
-	};
-
 	add = async (
 		file: FileData,
-		options?: { transaction?: AbstractTransaction; downloadRemote?: boolean },
 	): Promise<void> => {
 		let buffer = file.file ? await fileToArrayBuffer(file.file) : undefined;
-		if (!buffer && options?.downloadRemote && file.url) {
-			try {
-				buffer = await fetch(file.url, {
-					method: 'GET',
-					credentials: 'include',
-				}).then((r) => r.arrayBuffer());
-			} catch (err) {
-				console.error(
-					"Failed to download file to cache it locally. The file will still be available using its URL. Check the file server's CORS configuration.",
-					err,
-				);
-			}
-		}
+
 		await this.run(
 			'files',
 			(store) => {
@@ -65,19 +41,16 @@ export class IdbPersistenceFileDb
 					type: file.type,
 					url: file.url,
 					buffer,
+					timestamp: file.timestamp,
 				} satisfies StoredFileData);
 			},
 			{
 				mode: 'readwrite',
-				transaction: options?.transaction as IDBTransaction,
 			},
 		);
 	};
-	markUploaded = async (
-		id: string,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<void> => {
-		const current = await this.getFileRaw(id, options);
+	markUploaded = async (id: string): Promise<void> => {
+		const current = await this.getFileRaw(id);
 
 		if (!current) {
 			throw new Error('File is not in local database');
@@ -93,24 +66,17 @@ export class IdbPersistenceFileDb
 			},
 			{
 				mode: 'readwrite',
-				transaction: options?.transaction as IDBTransaction,
 			},
 		);
 	};
-	get = async (
-		fileId: string,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<PersistedFileData | null> => {
-		const raw = await this.getFileRaw(fileId, options);
+	get = async (fileId: string): Promise<PersistedFileData | null> => {
+		const raw = await this.getFileRaw(fileId);
 		if (!raw) {
 			return null;
 		}
 		return this.hydrateFileData(raw);
 	};
-	delete = (
-		fileId: string,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<void> => {
+	delete = (fileId: string): Promise<void> => {
 		return this.run<undefined>(
 			'files',
 			(store) => {
@@ -118,15 +84,11 @@ export class IdbPersistenceFileDb
 			},
 			{
 				mode: 'readwrite',
-				transaction: options?.transaction as IDBTransaction,
 			},
 		);
 	};
-	markPendingDelete = async (
-		fileId: string,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<void> => {
-		const current = await this.getFileRaw(fileId, options);
+	markPendingDelete = async (fileId: string): Promise<void> => {
+		const current = await this.getFileRaw(fileId);
 
 		if (!current) {
 			throw new Error('File is not in local database');
@@ -142,29 +104,23 @@ export class IdbPersistenceFileDb
 			},
 			{
 				mode: 'readwrite',
-				transaction: options?.transaction as IDBTransaction,
 			},
 		);
 	};
-	listUnsynced = async (options?: {
-		transaction?: AbstractTransaction;
-	}): Promise<PersistedFileData[]> => {
+	listUnsynced = async (): Promise<PersistedFileData[]> => {
 		const raw = await this.run<StoredFileData[]>(
 			'files',
 			(store) => {
 				return store.index('remote').getAll('false');
 			},
-			{ mode: 'readonly', transaction: options?.transaction as IDBTransaction },
+			{ mode: 'readonly' },
 		);
 		return raw.map(this.hydrateFileData);
 	};
-	resetSyncedStatusSince = async (
-		since: string | null,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<void> => {
-		const tx: IDBTransaction =
-			(options?.transaction as any) ??
-			this.createTransaction(['files'], { mode: 'readwrite' });
+	resetSyncedStatusSince = async (since: string | null): Promise<void> => {
+		const tx: IDBTransaction = this.createTransaction(['files'], {
+			mode: 'readwrite',
+		});
 		const raw = await this.run<StoredFileData[]>(
 			'files',
 			(store) => {
@@ -194,7 +150,6 @@ export class IdbPersistenceFileDb
 	};
 	iterateOverPendingDelete = (
 		iterator: (file: PersistedFileData, store: IDBObjectStore) => void,
-		options?: { transaction?: IDBTransaction },
 	): Promise<void> => {
 		return this.iterate<StoredFileData>(
 			'files',
@@ -208,7 +163,6 @@ export class IdbPersistenceFileDb
 			},
 			{
 				mode: 'readwrite',
-				transaction: options?.transaction as IDBTransaction,
 			},
 		);
 	};
@@ -223,6 +177,20 @@ export class IdbPersistenceFileDb
 			size: await getSizeOfObjectStore(this.db, 'files'),
 		};
 	};
+	loadFileContents = async (file: FileData, ctx: Context): Promise<Blob> => {
+		if (file.file) return file.file;
+		if (file.localPath) {
+			throw new Error('Local file paths are not supported in browser');
+		}
+		if (file.url) {
+			const response = await ctx.environment.fetch(file.url);
+			if (!response.ok) {
+				throw new Error(`Failed to download file: ${response.statusText}`);
+			}
+			return response.blob();
+		}
+		throw new Error('File is missing url, file, and localPath');
+	}
 
 	private hydrateFileData = (raw: StoredFileData): PersistedFileData => {
 		(raw as any).remote = raw.remote === 'true';