@verdant-web/store 2.2.0 → 2.3.1

package/src/migration/db.ts

@@ -0,0 +1,147 @@
+export async function getDatabaseVersion(
+	indexedDB: IDBFactory,
+	namespace: string,
+	version: number,
+	log?: (...args: any[]) => void,
+): Promise<number> {
+	function openAndGetVersion(
+		resolve: (res: [number, IDBDatabase]) => void,
+		reject: (err: Error) => void,
+	) {
+		let currentVersion: number;
+		let database: IDBDatabase;
+		const request = indexedDB.open(
+			[namespace, 'collections'].join('_'),
+			version,
+		);
+		request.onupgradeneeded = async (event) => {
+			currentVersion = event.oldVersion;
+			const transaction = request.transaction!;
+			database = request.result;
+			transaction.abort();
+		};
+		request.onsuccess = (event) => {
+			resolve([request.result.version, request.result]);
+		};
+		request.onblocked = (event) => {
+			// retry if blocked
+			log?.('Database blocked, waiting...');
+			// setTimeout(() => {
+			// 	openAndGetVersion(resolve, reject);
+			// }, 200);
+		};
+		request.onerror = (event) => {
+			// FIXME: this fails if the code is older than the local database
+			resolve([currentVersion!, database!]);
+		};
+	}
+	const [currentVersion, db] = await new Promise<[number, IDBDatabase]>(
+		openAndGetVersion,
+	);
+	await closeDatabase(db);
+	return currentVersion;
+}
+
+export async function closeDatabase(db: IDBDatabase) {
+	db.close();
+	// FIXME: this isn't right!!!!
+	await new Promise<void>((resolve) => resolve());
+}
+
+/**
+ * Upgrades the database to the given version, using the given upgrader function.
+ */
+export async function upgradeDatabase(
+	indexedDb: IDBFactory,
+	namespace: string,
+	version: number,
+	upgrader: (
+		transaction: IDBTransaction,
+		db: IDBDatabase,
+		event: IDBVersionChangeEvent,
+	) => void,
+	log?: (...args: any[]) => void,
+): Promise<void> {
+	function openAndUpgrade(resolve: () => void, reject: (err: Error) => void) {
+		const request = indexedDb.open(
+			[namespace, 'collections'].join('_'),
+			version,
+		);
+		let wasUpgraded = false;
+		request.onupgradeneeded = (event) => {
+			const transaction = request.transaction!;
+			upgrader(transaction, request.result, event);
+			wasUpgraded = true;
+		};
+		request.onsuccess = (event) => {
+			request.result.close();
+			if (wasUpgraded) {
+				resolve();
+			} else {
+				reject(
+					new Error(
+						'Database was not upgraded when a version change was expected',
+					),
+				);
+			}
+		};
+		request.onerror = (event) => {
+			reject(request.error || new Error('Unknown error'));
+		};
+		request.onblocked = (event) => {
+			log?.('Database upgrade blocked, waiting...');
+			// setTimeout(() => {
+			// 	openAndUpgrade(resolve, reject);
+			// }, 200);
+		};
+	}
+	return new Promise(openAndUpgrade);
+}
+
+export async function acquireLock(
+	namespace: string,
+	procedure: () => Promise<void>,
+) {
+	if (typeof navigator !== 'undefined' && navigator.locks) {
+		await navigator.locks.request(`verdant_migration_${namespace}`, procedure);
+	} else {
+		// TODO: is there a fallback?
+		await procedure();
+	}
+}
+
+export async function openDatabase(
+	indexedDb: IDBFactory,
+	namespace: string,
+	version: number,
+): Promise<IDBDatabase> {
+	const db = await new Promise<IDBDatabase>((resolve, reject) => {
+		const request = indexedDb.open(
+			[namespace, 'collections'].join('_'),
+			version,
+		);
+		request.onupgradeneeded = async (event) => {
+			const transaction = request.transaction!;
+			transaction.abort();
+
+			reject(
+				new Error('Migration error: database version changed while migrating'),
+			);
+		};
+		request.onsuccess = (event) => {
+			resolve(request.result);
+		};
+		request.onblocked = (event) => {
+			reject(new Error('Migration error: database blocked'));
+		};
+		request.onerror = (event) => {
+			reject(new Error('Migration error: database error'));
+		};
+	});
+
+	db.addEventListener('versionchange', (event) => {
+		db.close();
+	});
+
+	return db;
+}

package/src/migration/errors.ts

@@ -0,0 +1,7 @@
+export class MigrationPathError extends Error {
+	readonly name = 'MigrationPathError';
+
+	constructor(public readonly message: string) {
+		super(message);
+	}
+}

package/src/{openDocumentDatabase.ts → migration/openDatabase.ts}

@@ -5,21 +5,33 @@ import {
 	ObjectIdentifier,
 	StorageSchema,
 	addFieldDefaults,
+	assert,
 	assignIndexValues,
+	assignOid,
 	assignOidPropertiesToAllSubObjects,
 	assignOidsToAllSubObjects,
 	cloneDeep,
 	createOid,
 	decomposeOid,
 	diffToPatches,
+	getOid,
+	hasOid,
 	initialToPatches,
 	migrationRange,
 	removeOidPropertiesFromAllSubObjects,
 	removeOidsFromAllSubObjects,
 } from '@verdant-web/common';
-import { Context } from './context.js';
-import { Metadata } from './metadata/Metadata.js';
-import { findAllOids, findOneOid } from './queries2/dbQueries.js';
+import { Context } from '../context.js';
+import { Metadata } from '../metadata/Metadata.js';
+import { findAllOids, findOneOid } from '../queries2/dbQueries.js';
+import {
+	acquireLock,
+	closeDatabase,
+	getDatabaseVersion,
+	openDatabase,
+	upgradeDatabase,
+} from './db.js';
+import { getMigrationPath } from './paths.js';
 
 const globalIDB =
 	typeof window !== 'undefined' ? window.indexedDB : (undefined as any);
@@ -53,13 +65,11 @@ export async function openDocumentDatabase({
 		version,
 	);
 
-	const toRunVersions = migrationRange(currentVersion, version);
-	const toRun = toRunVersions.map((ver) =>
-		migrations.find((m) => m.version === ver),
-	);
-	if (toRun.some((m) => !m)) {
-		throw new Error(`No migration found for version(s) ${toRunVersions}`);
-	}
+	const toRun = getMigrationPath({
+		currentVersion,
+		targetVersion: version,
+		migrations,
+	});
 
 	if (toRun.length > 0) {
 		await acquireLock(context.namespace, async () => {
@@ -240,86 +250,6 @@ export async function openDocumentDatabase({
 	}
 }
 
-async function getDatabaseVersion(
-	indexedDB: IDBFactory,
-	namespace: string,
-	version: number,
-	log?: (...args: any[]) => void,
-): Promise<number> {
-	function openAndGetVersion(
-		resolve: (res: [number, IDBDatabase]) => void,
-		reject: (err: Error) => void,
-	) {
-		let currentVersion: number;
-		let database: IDBDatabase;
-		const request = indexedDB.open(
-			[namespace, 'collections'].join('_'),
-			version,
-		);
-		request.onupgradeneeded = async (event) => {
-			currentVersion = event.oldVersion;
-			const transaction = request.transaction!;
-			database = request.result;
-			transaction.abort();
-		};
-		request.onsuccess = (event) => {
-			resolve([request.result.version, request.result]);
-		};
-		request.onblocked = (event) => {
-			// retry if blocked
-			log?.('Database blocked, waiting...');
-			// setTimeout(() => {
-			// 	openAndGetVersion(resolve, reject);
-			// }, 200);
-		};
-		request.onerror = (event) => {
-			// FIXME: this fails if the code is older than the local database
-			resolve([currentVersion!, database!]);
-		};
-	}
-	const [currentVersion, db] = await new Promise<[number, IDBDatabase]>(
-		openAndGetVersion,
-	);
-	await closeDatabase(db);
-	return currentVersion;
-}
-
-async function openDatabase(
-	indexedDb: IDBFactory,
-	namespace: string,
-	version: number,
-): Promise<IDBDatabase> {
-	const db = await new Promise<IDBDatabase>((resolve, reject) => {
-		const request = indexedDb.open(
-			[namespace, 'collections'].join('_'),
-			version,
-		);
-		request.onupgradeneeded = async (event) => {
-			const transaction = request.transaction!;
-			transaction.abort();
-
-			reject(
-				new Error('Migration error: database version changed while migrating'),
-			);
-		};
-		request.onsuccess = (event) => {
-			resolve(request.result);
-		};
-		request.onblocked = (event) => {
-			reject(new Error('Migration error: database blocked'));
-		};
-		request.onerror = (event) => {
-			reject(new Error('Migration error: database error'));
-		};
-	});
-
-	db.addEventListener('versionchange', (event) => {
-		db.close();
-	});
-
-	return db;
-}
-
 function getMigrationMutations({
 	migration,
 	meta,
@@ -360,28 +290,21 @@ function getMigrationMutations({
 	}, {} as any);
 }
 
-function getMigrationEngine({
-	meta,
+function getMigrationQueries({
 	migration,
 	context,
+	meta,
 }: {
-	log?: (...args: any[]) => void;
-	migration: Migration;
-	meta: Metadata;
+	migration: Migration<any>;
 	context: Context;
-}): MigrationEngine<any, any> {
-	function getMigrationNow() {
-		return meta.time.zero(migration.version);
-	}
-
-	const newOids = new Array<ObjectIdentifier>();
-
-	const queries = migration.oldCollections.reduce((acc, collectionName) => {
+	meta: Metadata;
+}) {
+	return migration.oldCollections.reduce((acc, collectionName) => {
 		acc[collectionName] = {
 			get: async (id: string) => {
 				const oid = createOid(collectionName, id);
 				const doc = await meta.getDocumentSnapshot(oid);
-				removeOidsFromAllSubObjects(doc);
+				// removeOidsFromAllSubObjects(doc);
 				return doc;
 			},
 			findOne: async (filter: CollectionFilter) => {
@@ -392,7 +315,7 @@ function getMigrationEngine({
 				});
 				if (!oid) return null;
 				const doc = await meta.getDocumentSnapshot(oid);
-				removeOidsFromAllSubObjects(doc);
+				// removeOidsFromAllSubObjects(doc);
 				return doc;
 			},
 			findAll: async (filter: CollectionFilter) => {
@@ -404,12 +327,35 @@ function getMigrationEngine({
 				const docs = await Promise.all(
 					oids.map((oid) => meta.getDocumentSnapshot(oid)),
 				);
-				docs.forEach((doc) => removeOidsFromAllSubObjects(doc));
+				// docs.forEach((doc) => removeOidsFromAllSubObjects(doc));
 				return docs;
 			},
 		};
 		return acc;
 	}, {} as any);
+}
+
+function getMigrationEngine({
+	meta,
+	migration,
+	context,
+}: {
+	log?: (...args: any[]) => void;
+	migration: Migration;
+	meta: Metadata;
+	context: Context;
+}): MigrationEngine<any, any> {
+	function getMigrationNow() {
+		return meta.time.zero(migration.version);
+	}
+
+	const newOids = new Array<ObjectIdentifier>();
+
+	const queries = getMigrationQueries({
+		migration,
+		context,
+		meta,
+	});
 	const mutations = getMigrationMutations({
 		migration,
 		getMigrationNow,
@@ -418,45 +364,27 @@ function getMigrationEngine({
 	});
 	const awaitables = new Array<Promise<any>>();
 	const engine: MigrationEngine<StorageSchema, StorageSchema> = {
+		log: context.log,
 		newOids,
 		migrate: async (collection, strategy) => {
-			const docs = await new Promise<any[]>((resolve, reject) => {
-				const transaction = context.documentDb.transaction(
-					collection,
-					'readonly',
-				);
-
-				const store = transaction.objectStore(collection);
-				const cursorReq = store.openCursor();
-
-				const documentsToMigrate: any[] = [];
-
-				cursorReq.onsuccess = async (event) => {
-					const cursor = cursorReq.result;
-					if (cursor) {
-						documentsToMigrate.push(cursor.value);
-						cursor.continue();
-					} else {
-						resolve(documentsToMigrate);
-					}
-				};
-				cursorReq.onerror = (event) => {
-					reject(cursorReq.error);
-				};
-			});
+			const docs = await queries[collection].findAll();
 
 			await Promise.all(
-				docs.map(async (doc) => {
+				docs.map(async (doc: any) => {
+					assert(
+						hasOid(doc),
+						`Document is missing an OID: ${JSON.stringify(doc)}`,
+					);
 					const original = cloneDeep(doc);
 					// remove any indexes before computing the diff
-					const collectionSpec = migration.oldSchema.collections[collection];
-					const indexKeys = [
-						...Object.keys(collectionSpec.synthetics || {}),
-						...Object.keys(collectionSpec.compounds || {}),
-					];
-					indexKeys.forEach((key) => {
-						delete doc[key];
-					});
+					// const collectionSpec = migration.oldSchema.collections[collection];
+					// const indexKeys = [
+					// 	...Object.keys(collectionSpec.synthetics || {}),
+					// 	...Object.keys(collectionSpec.compounds || {}),
+					// ];
+					// indexKeys.forEach((key) => {
+					// 	delete doc[key];
+					// });
 					// @ts-ignore - excessive type resolution
 					const newValue = await strategy(doc);
 					if (newValue) {
@@ -493,6 +421,7 @@ function getMigrationEngine({
 function getVersion1MigrationEngine({
 	meta,
 	migration,
+	context,
 }: {
 	context: OpenDocumentDbContext;
 	migration: Migration;
@@ -519,6 +448,7 @@ function getVersion1MigrationEngine({
 		meta,
 	});
 	const engine: MigrationEngine<StorageSchema, StorageSchema> = {
+		log: context.log,
 		newOids,
 		migrate: (collection, strategy) => {
 			throw new Error(
@@ -532,59 +462,6 @@ function getVersion1MigrationEngine({
 	return engine;
 }
 
-async function closeDatabase(db: IDBDatabase) {
-	db.close();
-	// FIXME: this isn't right!!!!
-	await new Promise<void>((resolve) => resolve());
-}
-
-async function upgradeDatabase(
-	indexedDb: IDBFactory,
-	namespace: string,
-	version: number,
-	upgrader: (
-		transaction: IDBTransaction,
-		db: IDBDatabase,
-		event: IDBVersionChangeEvent,
-	) => void,
-	log?: (...args: any[]) => void,
-): Promise<void> {
-	function openAndUpgrade(resolve: () => void, reject: (err: Error) => void) {
-		const request = indexedDb.open(
-			[namespace, 'collections'].join('_'),
-			version,
-		);
-		let wasUpgraded = false;
-		request.onupgradeneeded = (event) => {
-			const transaction = request.transaction!;
-			upgrader(transaction, request.result, event);
-			wasUpgraded = true;
-		};
-		request.onsuccess = (event) => {
-			request.result.close();
-			if (wasUpgraded) {
-				resolve();
-			} else {
-				reject(
-					new Error(
-						'Database was not upgraded when a version change was expected',
-					),
-				);
-			}
-		};
-		request.onerror = (event) => {
-			reject(request.error || new Error('Unknown error'));
-		};
-		request.onblocked = (event) => {
-			log?.('Database upgrade blocked, waiting...');
-			// setTimeout(() => {
-			// 	openAndUpgrade(resolve, reject);
-			// }, 200);
-		};
-	}
-	return new Promise(openAndUpgrade);
-}
-
 async function getAllKeys(store: IDBObjectStore) {
 	return new Promise<IDBValidKey[]>((resolve, reject) => {
 		const request = store.getAllKeys();
@@ -620,12 +497,3 @@ async function putView(store: IDBObjectStore, view: any) {
 		};
 	});
 }
-
-async function acquireLock(namespace: string, procedure: () => Promise<void>) {
-	if (typeof navigator !== 'undefined' && navigator.locks) {
-		await navigator.locks.request(`lo-fi_migration_${namespace}`, procedure);
-	} else {
-		// TODO: is there a fallback?
-		await procedure();
-	}
-}

package/src/migration/paths.test.ts

@@ -0,0 +1,93 @@
+import { Migration } from '@verdant-web/common';
+import { describe, expect, it } from 'vitest';
+import { getMigrationPath } from './paths.js';
+
+function mockMigration(from: number, to: number) {
+	return {
+		oldSchema: {
+			version: from,
+		},
+		newSchema: {
+			version: to,
+		},
+	} as any as Migration;
+}
+
+describe('migration pathfinding', () => {
+	it('should follow a linear chain', () => {
+		const migrations = [
+			mockMigration(0, 1),
+			mockMigration(1, 2),
+			mockMigration(2, 3),
+			mockMigration(3, 4),
+			mockMigration(4, 5),
+		];
+		const path = getMigrationPath({
+			currentVersion: 0,
+			targetVersion: 5,
+			migrations,
+		});
+		expect(path).toEqual(migrations);
+	});
+	it('should choose the shortest path', () => {
+		const migrations = [
+			mockMigration(0, 1),
+			mockMigration(1, 2),
+			mockMigration(2, 3),
+			mockMigration(3, 4),
+			mockMigration(4, 5),
+			mockMigration(0, 5),
+		];
+		const path = getMigrationPath({
+			currentVersion: 0,
+			targetVersion: 5,
+			migrations,
+		});
+		expect(path).toEqual([mockMigration(0, 5)]);
+	});
+	it('should be resilient to dead ends', () => {
+		const migrations = [
+			mockMigration(0, 1),
+			mockMigration(1, 2),
+			mockMigration(4, 5),
+			mockMigration(5, 6),
+			mockMigration(1, 4),
+		];
+		const path = getMigrationPath({
+			currentVersion: 0,
+			targetVersion: 6,
+			migrations,
+		});
+		expect(path).toEqual([
+			mockMigration(0, 1),
+			mockMigration(1, 4),
+			mockMigration(4, 5),
+			mockMigration(5, 6),
+		]);
+	});
+	it('should error when no path exists to target', () => {
+		const migrations = [
+			mockMigration(0, 1),
+			mockMigration(2, 3),
+			mockMigration(3, 4),
+			mockMigration(4, 5),
+			mockMigration(5, 6),
+		];
+		expect(() => {
+			getMigrationPath({
+				currentVersion: 0,
+				targetVersion: 6,
+				migrations,
+			});
+		}).toThrow();
+	});
+	it('should return empty when versions are the same', () => {
+		expect(
+			getMigrationPath({
+				currentVersion: 1,
+				targetVersion: 1,
+				migrations: [],
+			}),
+		).toEqual([]);
+	});
+});

package/src/migration/paths.ts

@@ -0,0 +1,73 @@
+import { Migration } from '@verdant-web/common';
+import { MigrationPathError } from './errors.js';
+
+export function getMigrationPath({
+	currentVersion,
+	targetVersion,
+	migrations,
+}: {
+	currentVersion: number;
+	targetVersion: number;
+	migrations: Migration[];
+}) {
+	const path = getNextPathStep({
+		currentVersion,
+		targetVersion,
+		migrations,
+	});
+	if (!path) {
+		throw new MigrationPathError(
+			`No migration path found from ${currentVersion} to ${targetVersion}! This is a bug. If you're seeing this, contact the developer and provide them with the full contents of this message.`,
+		);
+	}
+	return path;
+}
+
+function getNextPathStep({
+	currentVersion,
+	targetVersion,
+	migrations,
+}: {
+	currentVersion: number;
+	targetVersion: number;
+	migrations: Migration[];
+}): Migration[] | null {
+	if (currentVersion === targetVersion) {
+		return [];
+	}
+
+	const fromHere = migrations
+		.filter((m) => m.oldSchema.version === currentVersion)
+		.sort((a, b) => b.newSchema.version - a.newSchema.version);
+
+	// keep trying next steps, starting from the largest step,
+	// until we find one that leads to the target version down the line
+	while (fromHere.length > 0) {
+		const next = fromHere.shift()!;
+		// this one goes too far (probably never relevant, but still)
+		if (next.newSchema.version > targetVersion) {
+			return null;
+		}
+		// exact match - we're done, return the path
+		if (next.newSchema.version === targetVersion) {
+			return [next];
+		}
+		// look ahead a down the line. do we reach the target? if so,
+		// we choose this path.
+		const nextPath = getNextPathStep({
+			currentVersion: next.newSchema.version,
+			targetVersion,
+			migrations,
+		});
+		if (nextPath) {
+			return [next, ...nextPath];
+		}
+
+		// otherwise, try the next one with a smaller increment
+	}
+
+	// no paths from here match at all! if another layer is calling this one,
+	// it will fallback to its next longest step. otherwise there may
+	// be no paths at all...
+	return null;
+}