@verdant-web/store 4.0.0 → 4.1.0-alpha.1

package/src/persistence/idb/queries/{IdbQueryDb.ts → IdbDocumentDb.ts}

@@ -6,23 +6,26 @@ import {
 	ObjectIdentifier,
 } from '@verdant-web/common';
 import { Context } from '../../../context/context.js';
-import {
-	AbstractTransaction,
-	CommonQueryOptions,
-	PersistenceQueryDb,
-	QueryMode,
-} from '../../interfaces.js';
+import { PersistenceDocumentDb } from '../../interfaces.js';
 import { IdbService } from '../IdbService.js';
 import { getRange } from './ranges.js';
 import { closeDatabase, getSizeOfObjectStore, isAbortError } from '../util.js';
 
-export class IdbQueryDb extends IdbService implements PersistenceQueryDb {
+export class IdbDocumentDb extends IdbService implements PersistenceDocumentDb {
 	private ctx;
-	constructor(db: IDBDatabase, context: Omit<Context, 'queries' | 'files'>) {
+	constructor(db: IDBDatabase, context: Omit<Context, 'documents' | 'files'>) {
 		super(db, { log: context.log });
 		this.ctx = context;
+		this.addDispose(() => {
+			this.ctx.log('info', 'Closing document database for', this.ctx.namespace);
+			return closeDatabase(this.db);
+		});
 	}
 
+	close = async () => {
+		await this.dispose();
+	};
+
 	stats = async (): Promise<
 		Record<string, { count: number; size: number }>
 	> => {
@@ -37,17 +40,6 @@ export class IdbQueryDb extends IdbService implements PersistenceQueryDb {
 		return collections;
 	};
 
-	transaction = (opts: {
-		mode?: QueryMode;
-		storeNames: string[];
-		abort?: AbortSignal;
-	}): AbstractTransaction => {
-		return this.createTransaction(opts.storeNames, {
-			mode: opts.mode,
-			abort: opts.abort,
-		});
-	};
-
 	findOneOid = async (opts: {
 		collection: string;
 		index?: CollectionFilter;
@@ -148,36 +140,15 @@ export class IdbQueryDb extends IdbService implements PersistenceQueryDb {
 
 	saveEntities = async (
 		entities: { oid: ObjectIdentifier; getSnapshot: () => any }[],
-		opts?: CommonQueryOptions & { abort?: AbortSignal },
+		optsAndInfo: { abort?: AbortSignal; collections: string[] },
 	): Promise<void> => {
-		if (entities.length === 0) return;
-
-		let collections = Array.from(
-			new Set(entities.map((e) => decomposeOid(e.oid).collection)),
-		);
-
-		const toRemove = collections.filter((c) => !this.ctx.schema.collections[c]);
-		if (toRemove.length > 0) {
-			this.ctx.log(
-				'warn',
-				`Ignoring entities from collections that no longer exist: ${toRemove.join(
-					', ',
-				)}`,
-			);
-		}
-		const withRemoved = new Set(collections);
-		toRemove.forEach((c) => withRemoved.delete(c));
-		collections = Array.from(withRemoved);
-
 		const options = {
-			transaction: this.createTransaction(collections, {
+			transaction: this.createTransaction(optsAndInfo.collections, {
 				mode: 'readwrite',
-				abort: opts?.abort,
+				abort: optsAndInfo.abort,
 			}),
 		};
 
-		// FIXME: not test is making it to this line
-
 		await Promise.all(
 			entities.map(async (e) => {
 				const snapshot = e.getSnapshot();
@@ -198,19 +169,11 @@ export class IdbQueryDb extends IdbService implements PersistenceQueryDb {
 			}),
 		);
 		options.transaction.commit();
-		this.ctx.entityEvents.emit('collectionsChanged', collections);
-		for (const entity of entities) {
-			this.ctx.entityEvents.emit('documentChanged', entity.oid);
-		}
 	};
 
-	reset = async (opts?: {
-		transaction?: AbstractTransaction;
-	}): Promise<void> => {
+	reset = async (): Promise<void> => {
 		const names = Object.keys(this.ctx.schema.collections);
-		const tx =
-			(opts?.transaction as IDBTransaction) ||
-			this.createTransaction(names, { mode: 'readwrite' });
+		const tx = this.createTransaction(names, { mode: 'readwrite' });
 		await Promise.all(
 			names.map((name) =>
 				this.run(name, (store) => store.clear(), { transaction: tx }),
@@ -220,10 +183,6 @@ export class IdbQueryDb extends IdbService implements PersistenceQueryDb {
 		this.ctx.log('info', '💨 Reset queryable storage');
 	};
 
-	dispose = () => {
-		return closeDatabase(this.db);
-	};
-
 	private saveDocument = async (
 		oid: ObjectIdentifier,
 		doc: any,
@@ -241,6 +200,7 @@ export class IdbQueryDb extends IdbService implements PersistenceQueryDb {
 			const schema = this.ctx.schema.collections[collection];
 			// no need to validate before storing; the entity's snapshot is already validated.
 			const indexes = getIndexValues(schema, doc);
+			indexes['@@@snapshot'] = JSON.stringify(doc);
 			await this.run(collection, (store) => store.put(indexes), {
 				mode: 'readwrite',
 				transaction,

package/src/persistence/idb/queries/migration/db.ts

@@ -3,22 +3,9 @@ import {
 	storeRequestPromise,
 	openDatabase as baseOpenDatabase,
 	getDocumentDbName,
+	closeDatabase,
 } from '../../util.js';
-import { OpenDocumentDbContext } from './types.js';
-
-export async function getDatabaseVersion(
-	indexedDB: IDBFactory,
-	namespace: string,
-): Promise<number> {
-	const databaseName = getDocumentDbName(namespace);
-	const dbInfo = await indexedDB.databases();
-	const existingDb = dbInfo.find((info) => info.name === databaseName);
-	if (existingDb) {
-		return existingDb.version ?? 0;
-	}
-
-	return 0;
-}
+import { Context } from '../../../../internal.js';
 
 /**
  * Upgrades the database to the given version, using the given upgrader function.
@@ -34,22 +21,22 @@ export async function upgradeDatabase(
 	) => void,
 	log?: (...args: any[]) => void,
 ): Promise<IDBDatabase> {
+	log?.('debug', 'Upgrading database', namespace, 'to version', version);
 	function openAndUpgrade(
 		resolve: (db: IDBDatabase) => void,
 		reject: (err: Error) => void,
 	) {
-		const request = indexedDb.open(
-			[namespace, 'collections'].join('_'),
-			version,
-		);
+		const request = indexedDb.open(getDocumentDbName(namespace), version);
 		let wasUpgraded = false;
 		request.onupgradeneeded = (event) => {
 			const transaction = request.transaction!;
 			upgrader(transaction, request.result, event);
 			wasUpgraded = true;
 		};
-		request.onsuccess = (event) => {
+		request.onsuccess = async (event) => {
 			if (wasUpgraded) {
+				// close the database
+				await closeDatabase(request.result);
 				resolve(request.result);
 			} else {
 				reject(
@@ -63,39 +50,29 @@ export async function upgradeDatabase(
 			reject(request.error || new Error('Unknown error'));
 		};
 		request.onblocked = (event) => {
-			log?.('Database upgrade blocked, waiting...');
-			// setTimeout(() => {
-			// 	openAndUpgrade(resolve, reject);
-			// }, 200);
+			log?.('Database upgrade blocked!');
+			reject(
+				new Error(
+					'Database upgrade blocked. The app may be open in another tab?',
+				),
+			);
 		};
 	}
 	return new Promise<IDBDatabase>(openAndUpgrade);
 }
 
-export async function acquireLock(
-	namespace: string,
-	procedure: () => Promise<void>,
-) {
-	if (typeof navigator !== 'undefined' && navigator.locks) {
-		await navigator.locks.request(`verdant_migration_${namespace}`, procedure);
-	} else {
-		// TODO: is there a fallback?
-		await procedure();
-	}
-}
-
 export async function openDatabase({
 	indexedDB = globalIDB,
 	namespace,
 	version,
-	context,
+	log,
 }: {
 	indexedDB?: IDBFactory;
 	namespace: string;
 	version: number;
-	context: OpenDocumentDbContext;
+	log?: Context['log'];
 }): Promise<IDBDatabase> {
-	context.log('debug', 'Opening database', namespace, 'at version', version);
+	log?.('debug', 'Opening database', namespace, 'at version', version);
 	const db = await baseOpenDatabase(
 		getDocumentDbName(namespace),
 		version,
@@ -106,6 +83,10 @@ export async function openDatabase({
 		db.close();
 	});
 
+	db.addEventListener('close', () => {
+		log?.('warn', 'Database closed', namespace);
+	});
+
 	return db;
 }
 

package/src/persistence/idb/util.ts

@@ -1,4 +1,5 @@
 import { roughSizeOfObject } from '@verdant-web/common';
+import { Context } from '../../internal.js';
 
 export const globalIDB =
 	typeof window !== 'undefined' ? window.indexedDB : (undefined as any);
@@ -182,38 +183,100 @@ export function createAbortableTransaction(
 }
 
 /**
- * Empties all data in a database without changing
- * its structure.
+ * Deletes any existing database with name `toName` and
+ * copies the index structure and all data
+ * from `from` to a new database.
+ *
+ * Does NOT run Verdant migrations. Use to copy existing
+ * data as-is.
  */
-export function emptyDatabase(db: IDBDatabase) {
-	const storeNames = Array.from(db.objectStoreNames);
-	const tx = db.transaction(storeNames, 'readwrite');
-	for (const storeName of storeNames) {
-		tx.objectStore(storeName).clear();
+export async function overwriteDatabase(
+	from: IDBDatabase,
+	toName: string,
+	ctx: Pick<Context, 'log'>,
+	indexedDB = window.indexedDB,
+) {
+	const databases = await getAllDatabaseNamesAndVersions(indexedDB);
+	if (databases.some((d) => d.name === toName)) {
+		await deleteDatabase(toName, indexedDB);
+		ctx.log('debug', 'Deleted existing database', toName);
 	}
-	return new Promise<void>((resolve, reject) => {
-		tx.oncomplete = () => resolve();
-		tx.onerror = () => reject(tx.error);
+
+	const to = await new Promise<IDBDatabase>((resolve, reject) => {
+		ctx.log('debug', 'Opening reset database', toName, 'at', from.version);
+		const openRequest = indexedDB.open(toName, from.version);
+		openRequest.onupgradeneeded = () => {
+			ctx.log(
+				'debug',
+				'Upgrading database',
+				toName,
+				'to version',
+				from.version,
+			);
+			// copy all indexes from original
+			const original = from;
+			const upgradeTx = openRequest.transaction;
+			if (!upgradeTx) {
+				throw new Error('No transaction');
+			}
+			for (const storeName of Array.from(original.objectStoreNames)) {
+				const originalObjectStore = original
+					.transaction(storeName)
+					.objectStore(storeName);
+				// create object store
+				upgradeTx.db.createObjectStore(storeName, {
+					keyPath: originalObjectStore.keyPath,
+					autoIncrement: originalObjectStore.autoIncrement,
+				});
+				const store = upgradeTx.objectStore(storeName);
+				const originalStore = original
+					.transaction(storeName)
+					.objectStore(storeName);
+				for (const index of Array.from(originalStore.indexNames)) {
+					const originalIndex = originalStore.index(index);
+					ctx.log('debug', 'Copying index', index);
+					store.createIndex(index, originalIndex.keyPath, {
+						unique: originalIndex.unique,
+						multiEntry: originalIndex.multiEntry,
+					});
+				}
+			}
+		};
+		openRequest.onsuccess = () => {
+			ctx.log('debug', 'Opened reset database', toName);
+			resolve(openRequest.result);
+		};
+		openRequest.onerror = () =>
+			reject(openRequest.error ?? new Error('Unknown database upgrade error'));
 	});
-}
 
-export async function copyDatabase(from: IDBDatabase, to: IDBDatabase) {
-	await emptyDatabase(to);
 	const records = await getAllFromObjectStores(
 		from,
 		Array.from(from.objectStoreNames),
 	);
-	const writeTx = to.transaction(Array.from(to.objectStoreNames), 'readwrite');
-	for (let i = 0; i < records.length; i++) {
-		const store = writeTx.objectStore(from.objectStoreNames[i]);
-		for (const record of records[i]) {
-			store.add(record);
+	await new Promise<void>((resolve, reject) => {
+		const writeTx = to.transaction(
+			Array.from(to.objectStoreNames),
+			'readwrite',
+		);
+		for (let i = 0; i < records.length; i++) {
+			const store = writeTx.objectStore(from.objectStoreNames[i]);
+			for (const record of records[i]) {
+				store.add(record);
+			}
 		}
-	}
-	return new Promise<void>((resolve, reject) => {
 		writeTx.oncomplete = () => resolve();
-		writeTx.onerror = () => reject(writeTx.error);
+		writeTx.onerror = (ev) => {
+			const err =
+				writeTx.error ??
+				(ev.target as any).transaction?.error ??
+				new Error('Unknown error');
+			ctx.log('critical', 'Error copying data', err);
+			reject(err);
+		};
 	});
+
+	await closeDatabase(to);
 }
 
 export function openDatabase(

package/src/persistence/interfaces.ts

@@ -2,20 +2,19 @@ import {
 	CollectionFilter,
 	DocumentBaseline,
 	FileData,
+	Migration,
 	ObjectIdentifier,
 	Operation,
 } from '@verdant-web/common';
 import { Context, InitialContext } from '../context/context.js';
 
 export interface AckInfo {
-	type: 'ack';
 	globalAckTimestamp: string | null;
 }
 
 export interface LocalReplicaInfo {
-	type: 'localReplicaInfo';
 	id: string;
-	userId: string | undefined;
+	userId: string | null;
 	ackedLogicalTime: string | null;
 	lastSyncedLogicalTime: string | null;
 }
@@ -37,90 +36,97 @@ export interface ExportedData {
 	files: File[];
 }
 
-export type AbstractTransaction = unknown;
+export type AbstractTransaction = any;
 export type QueryMode = 'readwrite' | 'readonly';
-export interface CommonQueryOptions {
-	transaction?: AbstractTransaction;
+export interface CommonQueryOptions<
+	Tx extends AbstractTransaction = AbstractTransaction,
+> {
+	transaction?: Tx;
 	mode?: QueryMode;
 }
 export type Iterator<T> = (item: T) => void | boolean;
 
-export interface PersistenceMetadataDb {
-	transaction(opts: {
-		mode?: QueryMode;
-		storeNames: string[];
-		abort?: AbortSignal;
-	}): AbstractTransaction;
-	dispose(): void | Promise<void>;
+export interface PersistenceMetadataDb<
+	Tx extends AbstractTransaction = AbstractTransaction,
+> {
+	transaction<T = void>(
+		opts: {
+			mode?: QueryMode;
+			storeNames: string[];
+			abort?: AbortSignal;
+		},
+		procedure: (tx: Tx) => Promise<T>,
+	): Promise<T>;
 
 	// infos
 	getAckInfo(): Promise<AckInfo>;
 	setGlobalAck(ack: string): Promise<void>;
-	getLocalReplica(opts?: CommonQueryOptions): Promise<LocalReplicaInfo>;
+	getLocalReplica(
+		opts?: CommonQueryOptions<Tx>,
+	): Promise<LocalReplicaInfo | undefined | null>;
 	updateLocalReplica(
-		data: Partial<LocalReplicaInfo>,
-		opts?: CommonQueryOptions,
+		data: LocalReplicaInfo,
+		opts?: CommonQueryOptions<Tx>,
 	): Promise<void>;
 
 	// baselines
 	iterateDocumentBaselines(
 		rootOid: string,
 		iterator: Iterator<DocumentBaseline>,
-		opts?: CommonQueryOptions,
+		opts?: CommonQueryOptions<Tx>,
 	): Promise<void>;
 	iterateCollectionBaselines(
 		collection: string,
 		iterator: Iterator<DocumentBaseline>,
-		opts?: CommonQueryOptions,
+		opts?: CommonQueryOptions<Tx>,
 	): Promise<void>;
 	iterateAllBaselines(
 		iterator: Iterator<DocumentBaseline>,
-		opts?: CommonQueryOptions,
+		opts?: CommonQueryOptions<Tx>,
 	): Promise<void>;
 	getBaseline(
 		oid: string,
-		opts?: CommonQueryOptions,
-	): Promise<DocumentBaseline>;
+		opts?: CommonQueryOptions<Tx>,
+	): Promise<DocumentBaseline | null>;
 	setBaselines(
 		baselines: DocumentBaseline[],
-		opts?: CommonQueryOptions,
+		opts?: CommonQueryOptions<Tx>,
 	): Promise<void>;
-	deleteBaseline(oid: string, opts?: CommonQueryOptions): Promise<void>;
+	deleteBaseline(oid: string, opts?: CommonQueryOptions<Tx>): Promise<void>;
 
 	// operations
 	iterateDocumentOperations(
 		rootOid: string,
 		iterator: Iterator<ClientOperation>,
-		opts?: CommonQueryOptions & {
+		opts?: CommonQueryOptions<Tx> & {
 			to?: string | null;
 		},
 	): Promise<void>;
 	iterateEntityOperations(
 		oid: string,
 		iterator: Iterator<ClientOperation>,
-		opts?: CommonQueryOptions & { to?: string | null },
+		opts?: CommonQueryOptions<Tx> & { to?: string | null },
 	): Promise<void>;
 	iterateCollectionOperations(
 		collection: string,
 		iterator: Iterator<ClientOperation>,
-		opts?: CommonQueryOptions,
+		opts?: CommonQueryOptions<Tx>,
 	): Promise<void>;
 	iterateLocalOperations(
 		iterator: Iterator<ClientOperation>,
-		opts?: CommonQueryOptions & {
+		opts?: CommonQueryOptions<Tx> & {
 			before?: string | null;
 			after?: string | null;
 		},
 	): Promise<void>;
 	/** Iterates over operations for an entity for processing and deletes them as it goes. */
-	consumeEntityOperations(
+	deleteEntityOperations(
 		oid: string,
-		iterator: Iterator<ClientOperation>,
-		opts?: CommonQueryOptions & { to?: string | null },
+		opts: CommonQueryOptions<Tx> & { to: string | null },
 	): Promise<void>;
 	iterateAllOperations(
 		iterator: Iterator<ClientOperation>,
-		opts?: CommonQueryOptions & {
+		opts?: CommonQueryOptions<Tx> & {
 			before?: string | null;
 			from?: string | null;
 		},
@@ -130,14 +136,11 @@ export interface PersistenceMetadataDb {
 	 */
 	addOperations(
 		ops: ClientOperation[],
-		opts?: CommonQueryOptions,
+		opts?: CommonQueryOptions<Tx>,
 	): Promise<ObjectIdentifier[]>;
 
 	/* WARNING: deletes all data */
-	reset(opts?: {
-		clearReplica?: boolean;
-		transaction?: AbstractTransaction;
-	}): Promise<void>;
+	reset(opts?: { clearReplica?: boolean; transaction?: Tx }): Promise<void>;
 
 	stats(): Promise<{
 		operationsSize: { count: number; size: number };
@@ -145,14 +148,7 @@ export interface PersistenceMetadataDb {
 	}>;
 }
 
-export interface PersistenceQueryDb {
-	transaction(opts: {
-		mode?: QueryMode;
-		storeNames: string[];
-		abort?: AbortSignal;
-	}): AbstractTransaction;
-	dispose(): void | Promise<void>;
-
+export interface PersistenceDocumentDb {
 	findOneOid(opts: {
 		collection: string;
 		index?: CollectionFilter;
@@ -166,12 +162,14 @@ export interface PersistenceQueryDb {
 
 	saveEntities(
 		entities: { oid: ObjectIdentifier; getSnapshot: () => any }[],
-		opts?: CommonQueryOptions & { abort?: AbortSignal },
+		optsAndInfo: { abort?: AbortSignal; collections: string[] },
 	): Promise<void>;
 
-	reset(opts?: { transaction?: AbstractTransaction }): Promise<void>;
+	reset(): Promise<void>;
 
 	stats(): Promise<Record<string, { count: number; size: number }>>;
+
+	close(): Promise<void>;
 }
 
 export interface PersistedFileData extends FileData {
@@ -179,62 +177,63 @@ export interface PersistedFileData extends FileData {
 }
 
 export interface PersistenceFileDb {
-	transaction(opts: {
-		mode?: QueryMode;
-		storeNames: string[];
-		abort?: AbortSignal;
-	}): AbstractTransaction;
-	dispose(): void | Promise<void>;
-
-	add(
-		file: FileData,
-		options?: { transaction?: AbstractTransaction; downloadRemote?: boolean },
-	): Promise<void>;
-	markUploaded(
-		fileId: string,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<void>;
-	get(
-		fileId: string,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<PersistedFileData | null>;
-	delete(
-		fileId: string,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<void>;
-	markPendingDelete(
-		fileId: string,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<void>;
-	listUnsynced(options?: {
-		transaction?: AbstractTransaction;
-	}): Promise<PersistedFileData[]>;
-	resetSyncedStatusSince(
-		since: string | null,
-		options?: { transaction?: AbstractTransaction },
-	): Promise<void>;
+	add(file: FileData, options?: { downloadRemote?: boolean }): Promise<void>;
+	markUploaded(fileId: string): Promise<void>;
+	get(fileId: string): Promise<PersistedFileData | null>;
+	delete(fileId: string): Promise<void>;
+	markPendingDelete(fileId: string): Promise<void>;
+	listUnsynced(): Promise<PersistedFileData[]>;
+	resetSyncedStatusSince(since: string | null): Promise<void>;
 	iterateOverPendingDelete(
-		iterator: (file: PersistedFileData, store: IDBObjectStore) => void,
-		options?: { transaction?: IDBTransaction },
+		iterator: (file: PersistedFileData) => void,
 	): Promise<void>;
-	getAll(options?: {
-		transaction?: AbstractTransaction;
-	}): Promise<PersistedFileData[]>;
+	loadFileContents(file: FileData, ctx: Context): Promise<Blob>;
+	getAll(): Promise<PersistedFileData[]>;
 	stats(): Promise<{ size: { count: number; size: number } }>;
 }
 
-export interface PersistenceImplementation {
+export interface PersistenceNamespace {
 	openMetadata(ctx: InitialContext): Promise<PersistenceMetadataDb>;
-	openQueries(ctx: Omit<Context, 'queries'>): Promise<PersistenceQueryDb>;
+	/**
+	 * Open the Documents database according to the schema in the given
+	 * context. By the time this is called with a version, relevant migrations
+	 * will have been applied.
+	 */
+	openDocuments(
+		ctx: Omit<Context, 'documents' | 'files'>,
+	): Promise<PersistenceDocumentDb>;
+	/**
+	 * Apply a migration to the namespace provided in ctx.
+	 * This should make any transformations necessary to the
+	 * document database to accommodate the new schema indexes.
+	 * The migration itself contains a lot of information about
+	 * what changed between versions.
+	 *
+	 * This method should also store the new version to persisted
+	 * metadata, however your implementation chooses to do that.
+	 */
+	applyMigration(ctx: InitialContext, migration: Migration<any>): Promise<void>;
 	openFiles(
-		ctx: Omit<Context, 'files' | 'queries'>,
+		ctx: Omit<Context, 'files' | 'documents'>,
 	): Promise<PersistenceFileDb>;
-	/** Copies all data (metadata/document queries/files) from one namespace to another. */
-	copyNamespace(from: string, to: string, ctx: InitialContext): Promise<void>;
+}
+
+export interface PersistenceImplementation {
+	name: string;
+	openNamespace(
+		namespace: string,
+		ctx: Pick<Context, 'log' | 'persistenceShutdownHandler'>,
+	): Promise<PersistenceNamespace>;
 	/** Returns a list of all persisted namespaces visible to this app. */
 	getNamespaces(): Promise<string[]>;
 	/** Deletes all data from a particular namespace. */
 	deleteNamespace(namespace: string, ctx: InitialContext): Promise<void>;
 	/** Gets the schema version of the given namespace */
 	getNamespaceVersion(namespace: string): Promise<number>;
+	/**
+	 * Copies all data from one namespace to another. It should
+	 * overwrite the target namespace such that data and database
+	 * schema are identical.
+	 */
+	copyNamespace(from: string, to: string, ctx: InitialContext): Promise<void>;
 }