@verdant-web/store 4.0.0 → 4.1.0-alpha.0

package/src/persistence/idb/idbPersistence.ts

@@ -1,22 +1,27 @@
 import { Context, InitialContext } from '../../context/context.js';
-import { PersistenceImplementation, PersistenceFileDb } from '../interfaces.js';
+import {
+	PersistenceImplementation,
+	PersistenceFileDb,
+	PersistenceNamespace,
+} from '../interfaces.js';
 import { IdbPersistenceFileDb } from './files/IdbPersistenceFileDb.js';
 import { IdbMetadataDb } from './metadata/IdbMetadataDb.js';
 import { openMetadataDatabase } from './metadata/openMetadataDatabase.js';
-import { IdbQueryDb } from './queries/IdbQueryDb.js';
-import { openQueryDatabase } from './queries/migration/openQueryDatabase.js';
-import { PersistenceMetadata } from '../PersistenceMetadata.js';
+import { IdbDocumentDb } from './queries/IdbDocumentDb.js';
 import {
 	closeDatabase,
 	deleteDatabase,
+	getDocumentDbName,
 	getMetadataDbName,
 	getNamespaceFromDatabaseInfo,
-	getSizesOfAllObjectStores,
+	overwriteDatabase,
 } from './util.js';
-import { getDatabaseVersion } from './queries/migration/db.js';
+import { openDatabase, upgradeDatabase } from './queries/migration/db.js';
+import { Migration } from '@verdant-web/common';
+import { OpenDocumentDbContext } from '../migration/types.js';
 
 export class IdbPersistence implements PersistenceImplementation {
-	private metadataDb: IDBDatabase | undefined;
+	name = 'IdbPersistence';
 	constructor(private indexedDB: IDBFactory = window.indexedDB) {}
 
 	getNamespaces = async (): Promise<string[]> => {
@@ -30,7 +35,14 @@ export class IdbPersistence implements PersistenceImplementation {
 	};
 
 	getNamespaceVersion = async (namespace: string): Promise<number> => {
-		return getDatabaseVersion(this.indexedDB, namespace);
+		const databaseName = getDocumentDbName(namespace);
+		const dbInfo = await this.indexedDB.databases();
+		const existingDb = dbInfo.find((info) => info.name === databaseName);
+		if (existingDb) {
+			return existingDb.version ?? 0;
+		}
+
+		return 0;
 	};
 
 	deleteNamespace = async (
@@ -43,8 +55,64 @@ export class IdbPersistence implements PersistenceImplementation {
 		]);
 	};
 
+	openNamespace = async (
+		namespace: string,
+	): Promise<IdbPersistenceNamespace> => {
+		return new IdbPersistenceNamespace(this.indexedDB, namespace);
+	};
+
+	copyNamespace = async (
+		from: string,
+		to: string,
+		ctx: InitialContext,
+	): Promise<void> => {
+		const fromCtx = { ...ctx, namespace: from };
+		const toCtx = { ...ctx, namespace: to };
+		const { db: fromMetaDb } = await openMetadataDatabase({
+			indexedDB: this.indexedDB,
+			log: fromCtx.log,
+			namespace: fromCtx.namespace,
+		});
+
+		// no need to involve files, as they store all data
+		// in the metadata database.
+
+		const fromDocumentsDb = await openDatabase({
+			indexedDB: this.indexedDB,
+			namespace: fromCtx.namespace,
+			version: fromCtx.schema.version,
+			log: fromCtx.log,
+		});
+
+		fromCtx.log(
+			'info',
+			`Copying data from ${fromCtx.namespace} to ${toCtx.namespace}`,
+		);
+
+		await overwriteDatabase(
+			fromMetaDb,
+			getMetadataDbName(toCtx.namespace),
+			toCtx,
+			this.indexedDB,
+		);
+		await overwriteDatabase(
+			fromDocumentsDb,
+			getDocumentDbName(toCtx.namespace),
+			toCtx,
+			this.indexedDB,
+		);
+
+		await closeDatabase(fromMetaDb);
+		await closeDatabase(fromDocumentsDb);
+	};
+}
+
+class IdbPersistenceNamespace implements PersistenceNamespace {
+	constructor(private indexedDB: IDBFactory, private namespace: string) {}
+	private metadataDb: IDBDatabase | undefined;
+
 	openFiles(
-		ctx: Omit<Context, 'files' | 'queries'>,
+		ctx: Omit<Context, 'files' | 'documents'>,
 	): Promise<PersistenceFileDb> {
 		if (!this.metadataDb) {
 			throw new Error(
@@ -58,59 +126,67 @@ export class IdbPersistence implements PersistenceImplementation {
 		const { db } = await openMetadataDatabase({
 			indexedDB: this.indexedDB,
 			log: ctx.log,
-			namespace: ctx.namespace,
+			namespace: this.namespace,
 		});
 		this.metadataDb = db;
+		ctx.persistenceShutdownHandler.register(() => closeDatabase(db));
 		return new IdbMetadataDb(db, ctx);
 	};
 
-	openQueries = async (ctx: InitialContext & { meta: PersistenceMetadata }) => {
-		const db = await openQueryDatabase({
+	openDocuments = async (ctx: OpenDocumentDbContext) => {
+		const db = await openDatabase({
 			version: ctx.schema.version,
 			indexedDB: this.indexedDB,
-			migrations: ctx.migrations,
-			context: ctx,
+			log: ctx.log,
+			namespace: this.namespace,
 		});
-		return new IdbQueryDb(db, ctx);
+		ctx.persistenceShutdownHandler.register(() => closeDatabase(db));
+		return new IdbDocumentDb(db, ctx);
 	};
 
-	copyNamespace = async (
-		from: string,
-		to: string,
+	applyMigration = async (
 		ctx: InitialContext,
+		migration: Migration<any>,
 	): Promise<void> => {
-		const fromCtx = { ...ctx, namespace: from, originalNamespace: from };
-		const fromMetaDb = await this.openMetadata(fromCtx);
-		const fromMeta = new PersistenceMetadata(fromMetaDb, fromCtx);
-		const fromQueries = await this.openQueries({ ...fromCtx, meta: fromMeta });
-		ctx.log('info', `Copying data from ${from} to ${to}`);
-
-		const { db: toMetaDb } = await openMetadataDatabase({
-			indexedDB: this.indexedDB,
-			log: ctx.log,
-			namespace: to,
-		});
-		ctx.log('debug', 'Metadata database opened');
-		await fromMetaDb.cloneTo(toMetaDb);
-		ctx.log('debug', 'Metadata copied');
-
-		const toQueryDb = await openQueryDatabase({
-			version: ctx.schema.version,
-			indexedDB: this.indexedDB,
-			migrations: ctx.migrations,
-			context: { ...ctx, namespace: to, originalNamespace: to, meta: fromMeta },
-		});
-		await fromQueries.cloneTo(toQueryDb);
-		ctx.log('debug', 'Indexes copied');
 		ctx.log(
 			'debug',
-			'New document store stats:',
-			await getSizesOfAllObjectStores(toQueryDb),
+			'Applying migration',
+			migration.newSchema.version,
+			migration,
 		);
+		await upgradeDatabase(
+			this.indexedDB,
+			this.namespace,
+			migration.newSchema.version,
+			(transaction, db) => {
+				for (const newCollection of migration.addedCollections) {
+					db.createObjectStore(newCollection, {
+						keyPath: migration.newSchema.collections[newCollection].primaryKey,
+						autoIncrement: false,
+					});
+				}
 
-		await fromMetaDb.dispose();
-		await closeDatabase(toMetaDb);
-		await fromQueries.dispose();
-		await closeDatabase(toQueryDb);
+				for (const collection of migration.allCollections) {
+					const store = transaction.objectStore(collection);
+					// apply new indexes
+					for (const newIndex of migration.addedIndexes[collection] || []) {
+						store.createIndex(newIndex.name, newIndex.name, {
+							multiEntry: newIndex.multiEntry,
+						});
+					}
+					// remove old indexes
+					for (const oldIndex of migration.removedIndexes[collection] || []) {
+						store.deleteIndex(oldIndex.name);
+					}
+				}
+				for (const removedCollection of migration.removedCollections) {
+					// !! can't delete the store, because old operations that relate to
+					// this store may still exist in history. instead, we can clear it out
+					// and leave it in place
+					transaction.objectStore(removedCollection).clear();
+				}
+			},
+			ctx.log,
+		);
 	};
 }

package/src/persistence/idb/metadata/IdbMetadataDb.ts

@@ -18,7 +18,6 @@ import {
 	PersistenceMetadataDb,
 } from '../../interfaces.js';
 import { IdbService } from '../IdbService.js';
-import cuid from 'cuid';
 import { closeDatabase, getSizeOfObjectStore } from '../util.js';
 import { Context } from '../../../context/context.js';
 
@@ -34,24 +33,35 @@ export type StoredSchema = {
 	schema: string;
 };
 
-export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
+export class IdbMetadataDb
+	extends IdbService
+	implements PersistenceMetadataDb<IDBTransaction>
+{
 	constructor(
 		db: IDBDatabase,
 		private ctx: Pick<Context, 'log' | 'namespace'>,
 	) {
 		super(db, ctx);
-		this.addDispose(() => closeDatabase(db));
+		this.addDispose(() => {
+			this.ctx.log('info', `Closing metadata DB for`, this.ctx.namespace);
+			return closeDatabase(db);
+		});
 	}
 
-	transaction = (opts: {
-		mode?: 'readwrite' | 'readonly';
-		storeNames: string[];
-		abort?: AbortSignal;
-	}) => {
-		return this.createTransaction(opts.storeNames, {
+	transaction = async <T>(
+		opts: {
+			mode?: 'readwrite' | 'readonly';
+			storeNames: string[];
+			abort?: AbortSignal;
+		},
+		procedure: (tx: IDBTransaction) => Promise<T>,
+	) => {
+		const tx = this.createTransaction(opts.storeNames, {
 			mode: opts.mode,
 			abort: opts.abort,
 		});
+		const result = await procedure(tx);
+		return result;
 	};
 
 	getAckInfo = async (): Promise<AckInfo> => {
@@ -61,7 +71,6 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 		} else {
 			return {
 				globalAckTimestamp: null,
-				type: 'ack',
 			};
 		}
 	};
@@ -74,62 +83,37 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 		);
 	};
 
-	private _creatingLocalReplica: Promise<void> | undefined;
-	private cachedLocalReplica: LocalReplicaInfo | undefined;
-
 	getLocalReplica = async (
 		opts?: CommonQueryOptions,
-	): Promise<LocalReplicaInfo> => {
-		if (this.cachedLocalReplica) {
-			return this.cachedLocalReplica;
-		}
-
-		const lookup = await this.run<LocalReplicaInfo>(
+	): Promise<LocalReplicaInfo | undefined> => {
+		return this.run<LocalReplicaInfo | undefined>(
 			'info',
 			(store) => store.get('localReplicaInfo'),
-			this.convertOpts(opts),
+			opts,
 		);
-
-		// not cached, not in db, create it
-		if (!lookup) {
-			// prevent a race condition if get() is called again while we are
-			// creating the replica info
-			if (!this._creatingLocalReplica) {
-				this._creatingLocalReplica = (async () => {
-					// create our own replica info now
-					const replicaId = cuid();
-					const replicaInfo: LocalReplicaInfo = {
-						type: 'localReplicaInfo',
-						id: replicaId,
-						userId: undefined,
-						ackedLogicalTime: null,
-						lastSyncedLogicalTime: null,
-					};
-					await this.run('info', (store) => store.put(replicaInfo), {
-						mode: 'readwrite',
-					});
-					this.cachedLocalReplica = replicaInfo;
-				})();
-			}
-			await this._creatingLocalReplica;
-
-			return this.getLocalReplica(opts);
-		}
-
-		this.cachedLocalReplica = lookup;
-		return lookup;
 	};
 
 	updateLocalReplica = async (
-		data: Partial<LocalReplicaInfo>,
-		opts: CommonQueryOptions = writeOpts,
+		data: LocalReplicaInfo,
+		opts?: CommonQueryOptions,
 	): Promise<void> => {
-		const localReplicaInfo = await this.getLocalReplica(opts);
-		Object.assign(localReplicaInfo, data);
-		await this.run('info', (store) => store.put(localReplicaInfo), {
-			mode: 'readwrite',
-		});
-		this.cachedLocalReplica = localReplicaInfo;
+		try {
+			await this.run(
+				'info',
+				(store) =>
+					store.put({
+						...data,
+						type: 'localReplicaInfo',
+					}),
+				{
+					mode: 'readwrite',
+					transaction: opts?.transaction,
+				},
+			);
+		} catch (e) {
+			this.ctx.log('critical', 'Error updating local replica', data, e);
+			throw e;
+		}
 	};
 
 	iterateDocumentBaselines = async (
@@ -151,7 +135,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 				];
 			},
 			iterator,
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -170,7 +154,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 				];
 			},
 			iterator,
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -182,7 +166,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 			'baselines',
 			(store) => store.index('timestamp').openCursor(),
 			iterator,
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -193,7 +177,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 		return this.run<DocumentBaseline>(
 			'baselines',
 			(store) => store.get(oid),
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -204,7 +188,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 		await this.runAll<any>(
 			'baselines',
 			(store) => baselines.map((b) => store.put(b)),
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -212,11 +196,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 		oid: string,
 		opts: CommonQueryOptions = writeOpts,
 	): Promise<void> => {
-		await this.run(
-			'baselines',
-			(store) => store.delete(oid),
-			this.convertOpts(opts),
-		);
+		await this.run('baselines', (store) => store.delete(oid), opts);
 	};
 
 	iterateDocumentOperations = (
@@ -237,7 +217,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 				return index.openCursor(range);
 			},
 			iterator,
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -262,14 +242,13 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 				return store.openCursor(range);
 			},
 			iterator,
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
-	consumeEntityOperations = (
+	deleteEntityOperations = (
 		oid: string,
-		iterator: Iterator<ClientOperation>,
-		opts: CommonQueryOptions & { to?: string | null } = writeOpts,
+		opts: CommonQueryOptions & { to: string | null },
 	): Promise<void> => {
 		return this.iterate<StoredClientOperation>(
 			'operations',
@@ -283,10 +262,9 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 				return store.openCursor(range);
 			},
 			(op, store) => {
-				iterator(op);
 				store.delete(op.oid_timestamp);
 			},
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -304,7 +282,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 				);
 			},
 			iterator,
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -359,7 +337,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 				return store.index('timestamp').openCursor(range, 'next');
 			},
 			iterator,
-			this.convertOpts(opts),
+			opts,
 		);
 	};
 
@@ -375,7 +353,7 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 					affected.add(getOidRoot(op.oid));
 					return store.put(this.addOperationIndexes(op));
 				}),
-			this.convertOpts(opts),
+			opts,
 		);
 		return Array.from(affected);
 	};
@@ -414,15 +392,27 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 				mode: 'readwrite',
 				transaction: tx,
 			});
+		} else {
+			const localInfo = await this.getLocalReplica({
+				transaction: tx,
+			});
+			if (localInfo) {
+				localInfo.ackedLogicalTime = null;
+				localInfo.lastSyncedLogicalTime = null;
+				await this.run(
+					'info',
+					(store) =>
+						store.put({
+							...localInfo,
+							type: 'localReplicaInfo',
+						}),
+					{
+						mode: 'readwrite',
+						transaction: tx,
+					},
+				);
+			}
 		}
-
-		const localInfo = await this.getLocalReplica();
-		localInfo.ackedLogicalTime = null;
-		localInfo.lastSyncedLogicalTime = null;
-		await this.run('info', (store) => store.put(localInfo), {
-			mode: 'readwrite',
-			transaction: tx,
-		});
 	};
 
 	private resetBaselines = async (tx: IDBTransaction) => {
@@ -443,18 +433,6 @@ export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
 			d_t: createCompoundIndexValue(getOidRoot(op.oid), op.timestamp) as string,
 		};
 	};
-
-	private convertOpts = (opts?: CommonQueryOptions) => {
-		if (opts?.transaction && !(opts.transaction instanceof IDBTransaction)) {
-			throw new Error(
-				`Invalid IndexedDB transaction. You cannot mix persistence providers. ${typeof opts.transaction}`,
-			);
-		}
-		return {
-			mode: opts?.mode,
-			transaction: opts?.transaction as IDBTransaction | undefined,
-		};
-	};
 }
 
 const writeOpts = { mode: 'readwrite' } as const;

package/src/persistence/idb/metadata/openMetadataDatabase.ts

@@ -1,9 +1,5 @@
 import { replaceLegacyOidsInObject } from '@verdant-web/common';
-import {
-	closeDatabase,
-	getMetadataDbName,
-	storeRequestPromise,
-} from '../util.js';
+import { getMetadataDbName } from '../util.js';
 import { Context } from '../../../context/context.js';
 
 const migrations = [version1, version2, version3, version4, version5, version6];
@@ -13,18 +9,16 @@ export function openMetadataDatabase({
 	indexedDB = window.indexedDB,
 	namespace,
 	log,
-	metadataVersion = CURRENT_METADATA_VERSION,
 }: {
 	indexedDB?: IDBFactory;
 	namespace: string;
 	log?: Context['log'];
-	metadataVersion?: number;
 }): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
 	return new Promise<{ wasInitialized: boolean; db: IDBDatabase }>(
 		(resolve, reject) => {
 			const request = indexedDB.open(
 				getMetadataDbName(namespace),
-				metadataVersion,
+				CURRENT_METADATA_VERSION,
 			);
 			let wasInitialized = false;
 			request.onupgradeneeded = async (event) => {
@@ -56,94 +50,6 @@ export function openMetadataDatabase({
 	);
 }
 
-export async function openWIPMetadataDatabase({
-	wipNamespace,
-	namespace,
-	indexedDB,
-	log,
-	metadataVersion,
-}: {
-	indexedDB?: IDBFactory;
-	namespace: string;
-	wipNamespace: string;
-	log?: (...args: any[]) => void;
-	metadataVersion?: number;
-}): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
-	const result = await openMetadataDatabase({
-		namespace: wipNamespace,
-		indexedDB,
-		log,
-		metadataVersion,
-	});
-
-	// this WIP database was already set up.
-	if (!result.wasInitialized) {
-		return result;
-	}
-
-	log?.('debug', 'Beginning copy of production metadata database to WIP');
-	// copy all data from production metadata database
-	const { db: prodDb } = await openMetadataDatabase({
-		namespace,
-		indexedDB,
-		log,
-		metadataVersion,
-	});
-
-	const tx = prodDb.transaction(
-		['baselines', 'operations', 'info'],
-		'readonly',
-	);
-	const [baselines, operations, info] = await Promise.all([
-		storeRequestPromise(tx.objectStore('baselines').getAll()),
-		storeRequestPromise(tx.objectStore('operations').getAll()),
-		storeRequestPromise(tx.objectStore('info').getAll()),
-	]);
-
-	const wipTx = result.db.transaction(
-		['baselines', 'operations', 'info'],
-		'readwrite',
-	);
-	const wipBaselines = wipTx.objectStore('baselines');
-	const wipOperations = wipTx.objectStore('operations');
-	const wipInfo = wipTx.objectStore('info');
-
-	for (const baseline of baselines) {
-		wipBaselines.put(baseline);
-	}
-	for (const operation of operations) {
-		wipOperations.put(operation);
-	}
-	for (const infoItem of info) {
-		wipInfo.put(infoItem);
-	}
-
-	await new Promise<void>((resolve, reject) => {
-		wipTx.oncomplete = () => {
-			resolve();
-		};
-		wipTx.onerror = (event) => {
-			reject(event);
-		};
-		wipTx.onabort = (event) => {
-			reject(event);
-		};
-	});
-
-	await closeDatabase(prodDb);
-
-	log?.(
-		'debug',
-		'Finished copy of production metadata database to WIP. Copied:',
-		baselines.length,
-		'baselines,',
-		operations.length,
-		'operations',
-	);
-
-	return result;
-}
-
 async function version1(db: IDBDatabase, tx: IDBTransaction) {
 	const baselinesStore = db.createObjectStore('baselines', {
 		keyPath: 'oid',