@verdant-web/store 2.5.8 → 2.7.0

package/src/migration/openDatabase.ts

@@ -7,23 +7,22 @@ import {
 	addFieldDefaults,
 	assert,
 	assignIndexValues,
-	assignOid,
 	assignOidPropertiesToAllSubObjects,
 	assignOidsToAllSubObjects,
 	cloneDeep,
 	createOid,
 	decomposeOid,
 	diffToPatches,
-	getOid,
+	getIndexValues,
 	getOidRoot,
 	hasOid,
 	initialToPatches,
-	migrationRange,
 	removeOidPropertiesFromAllSubObjects,
-	removeOidsFromAllSubObjects,
 } from '@verdant-web/common';
 import { Context } from '../context.js';
+import { storeRequestPromise } from '../idb.js';
 import { Metadata } from '../metadata/Metadata.js';
+import { ClientOperation } from '../metadata/OperationsStore.js';
 import { findAllOids, findOneOid } from '../queries/dbQueries.js';
 import {
 	acquireLock,
@@ -33,7 +32,6 @@ import {
 	upgradeDatabase,
 } from './db.js';
 import { getMigrationPath } from './paths.js';
-import { ClientOperation } from '../metadata/OperationsStore.js';
 
 const globalIDB =
 	typeof window !== 'undefined' ? window.indexedDB : (undefined as any);
@@ -53,6 +51,10 @@ export async function openDocumentDatabase({
 	meta: Metadata;
 	context: OpenDocumentDbContext;
 }) {
+	if (context.schema.wip) {
+		throw new Error('Cannot open a production client with a WIP schema!');
+	}
+
 	const currentVersion = await getDatabaseVersion(
 		indexedDB,
 		context.namespace,
@@ -61,6 +63,7 @@ export async function openDocumentDatabase({
 	);
 
 	context.log(
+		'debug',
 		'Current database version:',
 		currentVersion,
 		'target version:',
@@ -74,220 +77,360 @@ export async function openDocumentDatabase({
 	});
 
 	if (toRun.length > 0) {
-		await acquireLock(context.namespace, async () => {
-			// now the fun part
-			for (const migration of toRun as Migration<any>[]) {
-				// special case: if this is the version 1 migration, we have no pre-existing database
-				// to use for the migration.
-				let engine: MigrationEngine<any, any>;
-				// migrations from 0 (i.e. initial migrations) don't attempt to open an existing db
-				if (migration.oldSchema.version === 0) {
-					engine = getInitialMigrationEngine({
-						meta,
-						migration,
-						context,
-					});
-					await migration.migrate(engine);
-				} else {
-					// open the database with the current (old) version for this migration. this should
-					// align with the database's current version.
-					const originalDatabase = await openDatabase(
-						indexedDB,
-						context.namespace,
-						migration.oldSchema.version,
-					);
+		context.log(
+			'debug',
+			'Migrations to run:',
+			toRun.map((m) => m.version),
+		);
+		await runMigrations({ context, toRun, meta, indexedDB });
+	}
+	return openDatabase(indexedDB, context.namespace, version, context.log);
+}
 
-					// this will only write to our metadata store via operations!
-					engine = getMigrationEngine({
-						meta,
-						migration,
-						context: {
-							...context,
-							documentDb: originalDatabase,
-						},
-					});
-					try {
-						await migration.migrate(engine);
-						// wait on any out-of-band async operations to complete
-						await Promise.all(engine.awaitables);
-					} catch (err) {
-						context.log(
-							'critical',
-							`Migration failed (${migration.oldSchema.version} -> ${migration.newSchema.version})`,
-							err,
-						);
-						throw err;
-					}
+export async function openWIPDocumentDatabase({
+	version,
+	indexedDB = globalIDB,
+	migrations,
+	meta,
+	context,
+	wipNamespace,
+}: {
+	version: number;
+	migrations: Migration<any>[];
+	indexedDB?: IDBFactory;
+	meta: Metadata;
+	context: OpenDocumentDbContext;
+	wipNamespace: string;
+}) {
+	context.log('debug', 'Opening WIP database', wipNamespace);
+	const currentWIPVersion = await getDatabaseVersion(
+		indexedDB,
+		wipNamespace,
+		version,
+		context.log,
+	);
 
-					// now we have to open the database again with the next version and
-					// make the appropriate schema changes during the upgrade.
-					await closeDatabase(originalDatabase);
-				}
+	if (currentWIPVersion === version) {
+		context.log('info', `WIP schema is up-to-date; not refreshing database`);
+	} else {
+		context.log('info', `WIP schema is out-of-date; refreshing database`);
 
-				await upgradeDatabase(
-					indexedDB,
-					context.namespace,
-					migration.newSchema.version,
-					(transaction, db) => {
-						for (const newCollection of migration.addedCollections) {
-							db.createObjectStore(newCollection, {
-								keyPath:
-									migration.newSchema.collections[newCollection].primaryKey,
-								autoIncrement: false,
-							});
-						}
+		// first we need to copy the data from the production database to the WIP database
+		// at the current (non-wip) version.
 
-						for (const collection of migration.allCollections) {
-							const store = transaction.objectStore(collection);
-							// apply new indexes
-							for (const newIndex of migration.addedIndexes[collection] || []) {
-								store.createIndex(newIndex.name, newIndex.name, {
-									multiEntry: newIndex.multiEntry,
-								});
-							}
-							// remove old indexes
-							for (const oldIndex of migration.removedIndexes[collection] ||
-								[]) {
-								store.deleteIndex(oldIndex.name);
-							}
-						}
-						for (const removedCollection of migration.removedCollections) {
-							// !! can't delete the store, because old operations that relate to
-							// this store may still exist in history. instead, we can clear it out
-							// and leave it in place
-							transaction.objectStore(removedCollection).clear();
-						}
-					},
-					context.log,
+		const initialToRun = getMigrationPath({
+			currentVersion: currentWIPVersion,
+			targetVersion: version - 1,
+			migrations,
+		});
+
+		if (initialToRun.length > 0) {
+			await runMigrations({
+				context,
+				toRun: initialToRun,
+				meta,
+				indexedDB,
+				namespace: wipNamespace,
+			});
+
+			// now, we copy the data from the main database.
+			const mainDatabase = await openDatabase(
+				indexedDB,
+				context.namespace,
+				version - 1,
+				context.log,
+			);
+
+			const wipDatabase = await openDatabase(
+				indexedDB,
+				wipNamespace,
+				version - 1,
+				context.log,
+			);
+
+			// DOMStringList... doesn't have iterable... why
+			const mainDatabaseStoreNames = new Array<string>();
+			for (let i = 0; i < mainDatabase.objectStoreNames.length; i++) {
+				mainDatabaseStoreNames.push(mainDatabase.objectStoreNames[i]);
+			}
+
+			const copyFromTransaction = mainDatabase.transaction(
+				mainDatabaseStoreNames,
+				'readonly',
+			);
+			const copyFromStores = mainDatabaseStoreNames.map((name) =>
+				copyFromTransaction.objectStore(name),
+			);
+			const allObjects = await Promise.all(
+				copyFromStores.map((store) => storeRequestPromise(store.getAll())),
+			);
+
+			const copyToTransaction = wipDatabase.transaction(
+				mainDatabaseStoreNames,
+				'readwrite',
+			);
+			const copyToStores = mainDatabaseStoreNames.map((name) =>
+				copyToTransaction.objectStore(name),
+			);
+
+			for (let i = 0; i < copyToStores.length; i++) {
+				await Promise.all(
+					allObjects[i].map((obj) => {
+						return storeRequestPromise(copyToStores[i].put(obj));
+					}),
 				);
+			}
+		}
 
-				/**
-				 * In cases where operations from the future have been
-				 * received by this client, we may have created entire
-				 * documents in metadata which were not written to storage
-				 * because all of their operations were in the future (
-				 * i.e. in the next version). We have to find those documents
-				 * and also write their snapshots to storage, because they
-				 * won't be present in storage already to 'refresh,' so
-				 * if we don't analyze metadata for 'future' operations like
-				 * this, we won't know they exist.
-				 *
-				 * This led to behavior where the metadata would be properly
-				 * synced, but after upgrading the app and migrating, items
-				 * would be missing from findAll and findOne queries.
-				 */
-				const docsWithUnappliedMigrations =
-					await getDocsWithUnappliedMigrations({
-						meta,
-						currentVersion: migration.oldSchema.version,
-						newVersion: migration.newSchema.version,
-					});
+		const toRun = getMigrationPath({
+			currentVersion: version - 1,
+			targetVersion: version,
+			migrations,
+		});
 
-				// once the schema is ready, we can write back the migrated documents
-				const upgradedDatabase = await openDatabase(
+		if (toRun.length > 0) {
+			await runMigrations({
+				context,
+				toRun,
+				meta,
+				indexedDB,
+				namespace: wipNamespace,
+			});
+		}
+	}
+
+	return openDatabase(indexedDB, wipNamespace, version, context.log);
+}
+
+async function runMigrations({
+	context,
+	toRun,
+	meta,
+	indexedDB = globalIDB,
+	namespace = context.namespace,
+}: {
+	context: OpenDocumentDbContext;
+	toRun: Migration<any>[];
+	meta: Metadata;
+	indexedDB?: IDBFactory;
+	namespace?: string;
+}) {
+	await acquireLock(namespace, async () => {
+		// now the fun part
+		for (const migration of toRun) {
+			// special case: if this is the version 1 migration, we have no pre-existing database
+			// to use for the migration.
+			let engine: MigrationEngine;
+			// migrations from 0 (i.e. initial migrations) don't attempt to open an existing db
+			if (migration.oldSchema.version === 0) {
+				engine = getInitialMigrationEngine({
+					meta,
+					migration,
+					context,
+				});
+				await migration.migrate(engine);
+			} else {
+				// open the database with the current (old) version for this migration. this should
+				// align with the database's current version.
+				const originalDatabase = await openDatabase(
 					indexedDB,
-					context.namespace,
-					migration.newSchema.version,
+					namespace,
+					migration.oldSchema.version,
+					context.log,
 				);
-				for (const collection of migration.allCollections) {
-					// first step is to read in all the keys we need to rewrite
-					const documentReadTransaction = upgradedDatabase.transaction(
-						collection,
-						'readwrite',
-					);
-					const readStore = documentReadTransaction.objectStore(collection);
-					const keys = await getAllKeys(readStore);
-					// map the keys to OIDs
-					const oids = keys.map((key) => createOid(collection, `${key}`));
-					oids.push(
-						...engine.newOids.filter((oid) => {
-							return decomposeOid(oid).collection === collection;
-						}),
-						...docsWithUnappliedMigrations.filter((oid) => {
-							return decomposeOid(oid).collection === collection;
-						}),
-					);
 
-					const snapshots = await Promise.all(
-						oids.map(async (oid) => {
-							try {
-								const snap = await meta.getDocumentSnapshot(oid, {
-									to: meta.time.now(migration.newSchema.version),
-								});
-								return [oid, snap];
-							} catch (e) {
-								// this seems to happen with baselines/ops which are not fully
-								// cleaned up after deletion?
-								context.log(
-									'error',
-									'Could not regenerate snapshot during migration for oid',
-									oid,
-									'this document will not be preserved',
-									e,
-								);
-								return null;
-							}
-						}),
+				// this will only write to our metadata store via operations!
+				engine = getMigrationEngine({
+					meta,
+					migration,
+					context: {
+						...context,
+						documentDb: originalDatabase,
+					},
+				});
+				try {
+					await migration.migrate(engine);
+					// wait on any out-of-band async operations to complete
+					await Promise.all(engine.awaitables);
+				} catch (err) {
+					context.log(
+						'critical',
+						`Migration failed (${migration.oldSchema.version} -> ${migration.newSchema.version})`,
+						err,
 					);
+					throw err;
+				}
 
-					const views = snapshots
-						.filter((s): s is [string, any] => !!s)
-						.map(([oid, snapshot]) => {
-							if (!snapshot) return [oid, undefined];
-							const view = assignIndexValues(
-								migration.newSchema.collections[collection],
-								snapshot,
-							);
-							// TODO: remove the need for this by only storing index values!
-							assignOidPropertiesToAllSubObjects(view);
-							return [oid, view];
+				// now we have to open the database again with the next version and
+				// make the appropriate schema changes during the upgrade.
+				await closeDatabase(originalDatabase);
+			}
+
+			context.log(
+				'debug',
+				'Upgrading database',
+				namespace,
+				'to version',
+				migration.newSchema.version,
+			);
+			await upgradeDatabase(
+				indexedDB,
+				namespace,
+				migration.newSchema.version,
+				(transaction, db) => {
+					for (const newCollection of migration.addedCollections) {
+						db.createObjectStore(newCollection, {
+							keyPath:
+								migration.newSchema.collections[newCollection].primaryKey,
+							autoIncrement: false,
 						});
+					}
 
-					// now we can write the documents back
-					const documentWriteTransaction = upgradedDatabase.transaction(
-						collection,
-						'readwrite',
-					);
-					const writeStore = documentWriteTransaction.objectStore(collection);
-					await Promise.all(
-						views.map(([oid, view]) => {
-							if (view) {
-								return putView(writeStore, view);
-							} else {
-								const { id } = decomposeOid(oid);
-								return deleteView(writeStore, id);
-							}
-						}),
-					);
-				}
+					for (const collection of migration.allCollections) {
+						const store = transaction.objectStore(collection);
+						// apply new indexes
+						for (const newIndex of migration.addedIndexes[collection] || []) {
+							store.createIndex(newIndex.name, newIndex.name, {
+								multiEntry: newIndex.multiEntry,
+							});
+						}
+						// remove old indexes
+						for (const oldIndex of migration.removedIndexes[collection] || []) {
+							store.deleteIndex(oldIndex.name);
+						}
+					}
+					for (const removedCollection of migration.removedCollections) {
+						// !! can't delete the store, because old operations that relate to
+						// this store may still exist in history. instead, we can clear it out
+						// and leave it in place
+						transaction.objectStore(removedCollection).clear();
+					}
+				},
+				context.log,
+			);
+
+			/**
+			 * In cases where operations from the future have been
+			 * received by this client, we may have created entire
+			 * documents in metadata which were not written to storage
+			 * because all of their operations were in the future (
+			 * i.e. in the next version). We have to find those documents
+			 * and also write their snapshots to storage, because they
+			 * won't be present in storage already to 'refresh,' so
+			 * if we don't analyze metadata for 'future' operations like
+			 * this, we won't know they exist.
+			 *
+			 * This led to behavior where the metadata would be properly
+			 * synced, but after upgrading the app and migrating, items
+			 * would be missing from findAll and findOne queries.
+			 */
+			const docsWithUnappliedMigrations = await getDocsWithUnappliedMigrations({
+				meta,
+				currentVersion: migration.oldSchema.version,
+				newVersion: migration.newSchema.version,
+			});
 
-				await closeDatabase(upgradedDatabase);
-
-				context.log(`
-					⬆️ v${migration.newSchema.version} Migration complete. Here's the rundown:
-						- Added collections: ${migration.addedCollections.join(', ')}
-						- Removed collections: ${migration.removedCollections.join(', ')}
-						- Changed collections: ${migration.changedCollections.join(', ')}
-						- New indexes: ${Object.keys(migration.addedIndexes)
-							.map((col) =>
-								migration.addedIndexes[col].map((i) => `${col}.${i.name}`),
-							)
-							.flatMap((i) => i)
-							.join(', ')}
-						- Removed indexes: ${Object.keys(migration.removedIndexes)
-							.map((col) =>
-								migration.removedIndexes[col].map((i) => `${col}.${i.name}`),
-							)
-							.flatMap((i) => i)
-							.join(', ')}
-				`);
+			// once the schema is ready, we can write back the migrated documents
+			const upgradedDatabase = await openDatabase(
+				indexedDB,
+				namespace,
+				migration.newSchema.version,
+				context.log,
+			);
+			for (const collection of migration.allCollections) {
+				// first step is to read in all the keys we need to rewrite
+				const documentReadTransaction = upgradedDatabase.transaction(
+					collection,
+					'readwrite',
+				);
+				const readStore = documentReadTransaction.objectStore(collection);
+				const keys = await getAllKeys(readStore);
+				// map the keys to OIDs
+				const oids = keys.map((key) => createOid(collection, `${key}`));
+				oids.push(
+					...engine.newOids.filter((oid) => {
+						return decomposeOid(oid).collection === collection;
+					}),
+					...docsWithUnappliedMigrations.filter((oid) => {
+						return decomposeOid(oid).collection === collection;
+					}),
+				);
+
+				const snapshots = await Promise.all(
+					oids.map(async (oid) => {
+						try {
+							const snap = await meta.getDocumentSnapshot(oid);
+							return [oid, snap];
+						} catch (e) {
+							// this seems to happen with baselines/ops which are not fully
+							// cleaned up after deletion?
+							context.log(
+								'error',
+								'Could not regenerate snapshot during migration for oid',
+								oid,
+								'this document will not be preserved',
+								e,
+							);
+							return null;
+						}
+					}),
+				);
+
+				const views = snapshots
+					.filter((s): s is [string, any] => !!s)
+					.map(([oid, snapshot]) => {
+						if (!snapshot) return [oid, undefined];
+						const view = getIndexValues(
+							migration.newSchema.collections[collection],
+							snapshot,
+						);
+						return [oid, view];
+					});
+
+				// now we can write the documents back
+				const documentWriteTransaction = upgradedDatabase.transaction(
+					collection,
+					'readwrite',
+				);
+				const writeStore = documentWriteTransaction.objectStore(collection);
+				await Promise.all(
+					views.map(([oid, view]) => {
+						if (view) {
+							return putView(writeStore, view).catch((err) => {
+								view;
+								throw err;
+							});
+						} else {
+							const { id } = decomposeOid(oid);
+							return deleteView(writeStore, id);
+						}
+					}),
+				);
 			}
-		});
-		return openDatabase(indexedDB, context.namespace, version);
-	} else {
-		// just open the database
-		return openDatabase(indexedDB, context.namespace, version);
-	}
+
+			await closeDatabase(upgradedDatabase);
+
+			context.log('debug', `Migration of ${namespace} complete.`);
+			context.log(`
+				⬆️ v${migration.newSchema.version} Migration complete. Here's the rundown:
+					- Added collections: ${migration.addedCollections.join(', ')}
+					- Removed collections: ${migration.removedCollections.join(', ')}
+					- Changed collections: ${migration.changedCollections.join(', ')}
+					- New indexes: ${Object.keys(migration.addedIndexes)
+						.map((col) =>
+							migration.addedIndexes[col].map((i) => `${col}.${i.name}`),
+						)
+						.flatMap((i) => i)
+						.join(', ')}
+					- Removed indexes: ${Object.keys(migration.removedIndexes)
+						.map((col) =>
+							migration.removedIndexes[col].map((i) => `${col}.${i.name}`),
+						)
+						.flatMap((i) => i)
+						.join(', ')}
+			`);
+		}
+	});
 }
 
 function getMigrationMutations({
@@ -347,7 +490,6 @@ function getMigrationQueries({
 					// only get the snapshot up to the previous version (newer operations may have synced)
 					to: meta.time.now(migration.oldSchema.version),
 				});
-				// removeOidsFromAllSubObjects(doc);
 				return doc;
 			},
 			findOne: async (filter: CollectionFilter) => {
@@ -361,7 +503,6 @@ function getMigrationQueries({
 					// only get the snapshot up to the previous version (newer operations may have synced)
 					to: meta.time.now(migration.oldSchema.version),
 				});
-				// removeOidsFromAllSubObjects(doc);
 				return doc;
 			},
 			findAll: async (filter: CollectionFilter) => {
@@ -378,7 +519,6 @@ function getMigrationQueries({
 						}),
 					),
 				);
-				// docs.forEach((doc) => removeOidsFromAllSubObjects(doc));
 				return docs;
 			},
 		};
@@ -395,7 +535,7 @@ function getMigrationEngine({
 	migration: Migration;
 	meta: Metadata;
 	context: Context;
-}): MigrationEngine<any, any> {
+}): MigrationEngine {
 	function getMigrationNow() {
 		return meta.time.zero(migration.version);
 	}
@@ -414,7 +554,7 @@ function getMigrationEngine({
 		meta,
 	});
 	const awaitables = new Array<Promise<any>>();
-	const engine: MigrationEngine<StorageSchema, StorageSchema> = {
+	const engine: MigrationEngine = {
 		log: context.log,
 		newOids,
 		migrate: async (collection, strategy) => {
@@ -427,15 +567,6 @@ function getMigrationEngine({
 						`Document is missing an OID: ${JSON.stringify(doc)}`,
 					);
 					const original = cloneDeep(doc);
-					// remove any indexes before computing the diff
-					// const collectionSpec = migration.oldSchema.collections[collection];
-					// const indexKeys = [
-					// 	...Object.keys(collectionSpec.synthetics || {}),
-					// 	...Object.keys(collectionSpec.compounds || {}),
-					// ];
-					// indexKeys.forEach((key) => {
-					// 	delete doc[key];
-					// });
 					// @ts-ignore - excessive type resolution
 					const newValue = await strategy(doc);
 					if (newValue) {
@@ -477,7 +608,7 @@ function getInitialMigrationEngine({
 	context: OpenDocumentDbContext;
 	migration: Migration;
 	meta: Metadata;
-}): MigrationEngine<any, any> {
+}): MigrationEngine {
 	function getMigrationNow() {
 		return meta.time.zero(migration.version);
 	}
@@ -498,7 +629,7 @@ function getInitialMigrationEngine({
 		newOids,
 		meta,
 	});
-	const engine: MigrationEngine<StorageSchema, StorageSchema> = {
+	const engine: MigrationEngine = {
 		log: context.log,
 		newOids,
 		migrate: () => {