@verdant-web/store 3.8.0 → 3.8.2

package/src/idb.ts

@@ -1,5 +1,8 @@
 import { roughSizeOfObject } from '@verdant-web/common';
 
+export const globalIDB =
+	typeof window !== 'undefined' ? window.indexedDB : (undefined as any);
+
 export function isAbortError(err: unknown) {
 	return err instanceof Error && err.name === 'AbortError';
 }
@@ -109,7 +112,7 @@ export async function closeDatabase(db: IDBDatabase) {
 
 export async function deleteAllDatabases(
 	namespace: string,
-	indexedDB: IDBFactory = window.indexedDB,
+	indexedDB: IDBFactory = globalIDB,
 ) {
 	const req1 = indexedDB.deleteDatabase([namespace, 'meta'].join('_'));
 	const req2 = indexedDB.deleteDatabase([namespace, 'collections'].join('_'));
@@ -163,3 +166,19 @@ export function createAbortableTransaction(
 	}
 	return tx;
 }
+
+/**
+ * Empties all data in a database without changing
+ * its structure.
+ */
+export function emptyDatabase(db: IDBDatabase) {
+	const storeNames = Array.from(db.objectStoreNames);
+	const tx = db.transaction(storeNames, 'readwrite');
+	for (const storeName of storeNames) {
+		tx.objectStore(storeName).clear();
+	}
+	return new Promise<void>((resolve, reject) => {
+		tx.oncomplete = () => resolve();
+		tx.onerror = () => reject(tx.error);
+	});
+}

package/src/metadata/openMetadataDatabase.ts

@@ -1,7 +1,7 @@
 import { replaceLegacyOidsInObject } from '@verdant-web/common';
 import { closeDatabase, storeRequestPromise } from '../idb.js';
 
-const migrations = [version1, version2, version3, version4, version5];
+const migrations = [version1, version2, version3, version4, version5, version6];
 
 export function openMetadataDatabase({
 	indexedDB = window.indexedDB,
@@ -258,3 +258,9 @@ async function version5(db: IDBDatabase, tx: IDBTransaction) {
 		};
 	});
 }
+
+// version 5->6: add timestamp to file metadata
+async function version6(db: IDBDatabase, tx: IDBTransaction) {
+	const files = tx.objectStore('files');
+	files.createIndex('timestamp', 'timestamp');
+}

package/src/migration/db.ts

@@ -1,3 +1,6 @@
+import { closeDatabase, globalIDB, storeRequestPromise } from '../idb.js';
+import { OpenDocumentDbContext } from './types.js';
+
 export async function getDatabaseVersion(
 	indexedDB: IDBFactory,
 	namespace: string,
@@ -42,12 +45,6 @@ export async function getDatabaseVersion(
 	return currentVersion;
 }
 
-export async function closeDatabase(db: IDBDatabase) {
-	db.close();
-	// FIXME: this isn't right!!!!
-	await new Promise<void>((resolve) => resolve());
-}
-
 /**
  * Upgrades the database to the given version, using the given upgrader function.
  */
@@ -61,8 +58,11 @@ export async function upgradeDatabase(
 		event: IDBVersionChangeEvent,
 	) => void,
 	log?: (...args: any[]) => void,
-): Promise<void> {
-	function openAndUpgrade(resolve: () => void, reject: (err: Error) => void) {
+): Promise<IDBDatabase> {
+	function openAndUpgrade(
+		resolve: (db: IDBDatabase) => void,
+		reject: (err: Error) => void,
+	) {
 		const request = indexedDb.open(
 			[namespace, 'collections'].join('_'),
 			version,
@@ -74,9 +74,8 @@ export async function upgradeDatabase(
 			wasUpgraded = true;
 		};
 		request.onsuccess = (event) => {
-			request.result.close();
 			if (wasUpgraded) {
-				resolve();
+				resolve(request.result);
 			} else {
 				reject(
 					new Error(
@@ -95,7 +94,7 @@ export async function upgradeDatabase(
 			// }, 200);
 		};
 	}
-	return new Promise(openAndUpgrade);
+	return new Promise<IDBDatabase>(openAndUpgrade);
 }
 
 export async function acquireLock(
@@ -110,15 +109,20 @@ export async function acquireLock(
 	}
 }
 
-export async function openDatabase(
-	indexedDb: IDBFactory,
-	namespace: string,
-	version: number,
-	log?: (...args: any[]) => void,
-): Promise<IDBDatabase> {
-	log?.('debug', 'Opening database', namespace, 'at version', version);
+export async function openDatabase({
+	indexedDB = globalIDB,
+	namespace,
+	version,
+	context,
+}: {
+	indexedDB?: IDBFactory;
+	namespace: string;
+	version: number;
+	context: OpenDocumentDbContext;
+}): Promise<IDBDatabase> {
+	context.log('debug', 'Opening database', namespace, 'at version', version);
 	const db = await new Promise<IDBDatabase>((resolve, reject) => {
-		const request = indexedDb.open(
+		const request = indexedDB.open(
 			[namespace, 'collections'].join('_'),
 			version,
 		);
@@ -126,7 +130,7 @@ export async function openDatabase(
 			const transaction = request.transaction!;
 			transaction.abort();
 
-			log?.(
+			context.log(
 				'error',
 				'Database upgrade needed, but not expected',
 				'Expected',
@@ -158,3 +162,41 @@ export async function openDatabase(
 
 	return db;
 }
+
+export async function copyAll(
+	sourceDatabase: IDBDatabase,
+	targetDatabase: IDBDatabase,
+) {
+	// DOMStringList... doesn't have iterable... why
+	const sourceStoreNames = new Array<string>();
+	for (let i = 0; i < sourceDatabase.objectStoreNames.length; i++) {
+		sourceStoreNames.push(sourceDatabase.objectStoreNames[i]);
+	}
+
+	const copyFromTransaction = sourceDatabase.transaction(
+		sourceStoreNames,
+		'readonly',
+	);
+	const copyFromStores = sourceStoreNames.map((name) =>
+		copyFromTransaction.objectStore(name),
+	);
+	const allObjects = await Promise.all(
+		copyFromStores.map((store) => storeRequestPromise(store.getAll())),
+	);
+
+	const copyToTransaction = targetDatabase.transaction(
+		sourceStoreNames,
+		'readwrite',
+	);
+	const copyToStores = sourceStoreNames.map((name) =>
+		copyToTransaction.objectStore(name),
+	);
+
+	for (let i = 0; i < copyToStores.length; i++) {
+		await Promise.all(
+			allObjects[i].map((obj) => {
+				return storeRequestPromise(copyToStores[i].put(obj));
+			}),
+		);
+	}
+}

package/src/migration/engine.ts

@@ -0,0 +1,248 @@
+import {
+	CollectionFilter,
+	Migration,
+	MigrationEngine,
+	ObjectIdentifier,
+	addFieldDefaults,
+	assert,
+	assignOidsToAllSubObjects,
+	cloneDeep,
+	createOid,
+	diffToPatches,
+	getOid,
+	initialToPatches,
+	removeOidPropertiesFromAllSubObjects,
+} from '@verdant-web/common';
+import { Context } from '../context.js';
+import { Metadata } from '../metadata/Metadata.js';
+import { findAllOids, findOneOid } from '../queries/dbQueries.js';
+import { OpenDocumentDbContext } from './types.js';
+
+function getMigrationMutations({
+	migration,
+	meta,
+	getMigrationNow,
+	newOids,
+}: {
+	migration: Migration<any>;
+	newOids: string[];
+	getMigrationNow: () => string;
+	meta: Metadata;
+}) {
+	return migration.allCollections.reduce((acc, collectionName) => {
+		acc[collectionName] = {
+			put: async (doc: any) => {
+				// add defaults
+				addFieldDefaults(migration.newSchema.collections[collectionName], doc);
+				const primaryKey =
+					doc[migration.newSchema.collections[collectionName].primaryKey];
+				const oid = createOid(collectionName, primaryKey);
+				newOids.push(oid);
+				await meta.insertLocalOperations(
+					initialToPatches(doc, oid, getMigrationNow),
+				);
+				return doc;
+			},
+			delete: async (id: string) => {
+				const rootOid = createOid(collectionName, id);
+				const allOids = await meta.getAllDocumentRelatedOids(rootOid);
+				return meta.insertLocalOperations(
+					allOids.map((oid) => ({
+						oid,
+						timestamp: getMigrationNow(),
+						data: { op: 'delete' },
+					})),
+				);
+			},
+		};
+		return acc;
+	}, {} as any);
+}
+
+function getMigrationQueries({
+	migration,
+	context,
+	meta,
+}: {
+	migration: Migration<any>;
+	context: Context;
+	meta: Metadata;
+}) {
+	return migration.oldCollections.reduce((acc, collectionName) => {
+		acc[collectionName] = {
+			get: async (id: string) => {
+				const oid = createOid(collectionName, id);
+				const doc = await meta.getDocumentSnapshot(oid, {
+					// only get the snapshot up to the previous version (newer operations may have synced)
+					to: meta.time.now(migration.oldSchema.version),
+				});
+				return doc;
+			},
+			findOne: async (filter: CollectionFilter) => {
+				const oid = await findOneOid({
+					collection: collectionName,
+					index: filter,
+					context,
+				});
+				if (!oid) return null;
+				const doc = await meta.getDocumentSnapshot(oid, {
+					// only get the snapshot up to the previous version (newer operations may have synced)
+					to: meta.time.now(migration.oldSchema.version),
+				});
+				return doc;
+			},
+			findAll: async (filter: CollectionFilter) => {
+				const oids = await findAllOids({
+					collection: collectionName,
+					index: filter,
+					context,
+				});
+				const docs = await Promise.all(
+					oids.map((oid) =>
+						meta.getDocumentSnapshot(oid, {
+							// only get the snapshot up to the previous version (newer operations may have synced)
+							to: meta.time.now(migration.oldSchema.version),
+						}),
+					),
+				);
+				return docs;
+			},
+		};
+		return acc;
+	}, {} as any);
+}
+
+export function getMigrationEngine({
+	meta,
+	migration,
+	context,
+}: {
+	log?: (...args: any[]) => void;
+	migration: Migration;
+	meta: Metadata;
+	context: Context;
+}): MigrationEngine {
+	function getMigrationNow() {
+		return meta.time.zero(migration.version);
+	}
+
+	const newOids = new Array<ObjectIdentifier>();
+
+	const queries = getMigrationQueries({
+		migration,
+		context,
+		meta,
+	});
+	const mutations = getMigrationMutations({
+		migration,
+		getMigrationNow,
+		newOids,
+		meta,
+	});
+	const deleteCollection = async (collection: string) => {
+		const allOids = await meta.getAllCollectionRelatedOids(collection);
+		return meta.insertLocalOperations(
+			allOids.map((oid) => ({
+				oid,
+				timestamp: getMigrationNow(),
+				data: { op: 'delete' },
+			})),
+		);
+	};
+	const awaitables = new Array<Promise<any>>();
+	const engine: MigrationEngine = {
+		log: context.log,
+		newOids,
+		deleteCollection,
+		migrate: async (collection, strategy) => {
+			const docs = await queries[collection].findAll();
+
+			await Promise.all(
+				docs.filter(Boolean).map(async (doc: any) => {
+					const rootOid = getOid(doc);
+					assert(
+						!!rootOid,
+						`Document is missing an OID: ${JSON.stringify(doc)}`,
+					);
+					const original = cloneDeep(doc);
+					// @ts-ignore - excessive type resolution
+					const newValue = await strategy(doc);
+					if (newValue) {
+						// the migration has altered the shape of our document. we need
+						// to create the operation from the diff and write it to meta as
+						// a migration patch
+						removeOidPropertiesFromAllSubObjects(original);
+						removeOidPropertiesFromAllSubObjects(newValue);
+						assignOidsToAllSubObjects(newValue);
+						const patches = diffToPatches(
+							original,
+							newValue,
+							getMigrationNow,
+							undefined,
+							[],
+							{
+								mergeUnknownObjects: true,
+							},
+						);
+						if (patches.length > 0) {
+							await meta.insertLocalOperations(patches);
+						}
+					}
+				}),
+			);
+		},
+		queries,
+		mutations,
+		awaitables,
+	};
+	return engine;
+}
+
+export function getInitialMigrationEngine({
+	meta,
+	migration,
+	context,
+}: {
+	context: OpenDocumentDbContext;
+	migration: Migration;
+	meta: Metadata;
+}): MigrationEngine {
+	function getMigrationNow() {
+		return meta.time.zero(migration.version);
+	}
+
+	const newOids = new Array<ObjectIdentifier>();
+
+	const queries = new Proxy({} as any, {
+		get() {
+			throw new Error(
+				'Queries are not available in initial migrations; there is no database yet!',
+			);
+		},
+	}) as any;
+
+	const mutations = getMigrationMutations({
+		migration,
+		getMigrationNow,
+		newOids,
+		meta,
+	});
+	const engine: MigrationEngine = {
+		log: context.log,
+		newOids,
+		deleteCollection: () => {
+			throw new Error(
+				'Calling deleteCollection() in initial migrations is not supported! Use initial migrations to seed initial data using mutations.',
+			);
+		},
+		migrate: () => {
+			throw new Error(
+				'Calling migrate() in initial migrations is not supported! Use initial migrations to seed initial data using mutations.',
+			);
+		},
+		queries,
+		mutations,
+		awaitables: [],
+	};
+	return engine;
+}