@verdant-web/store 4.0.0 → 4.1.0-alpha.0

package/src/persistence/{idb/queries/migration → migration}/engine.ts

@@ -14,7 +14,7 @@ import {
 	AuthorizationKey,
 } from '@verdant-web/common';
 import { OpenDocumentDbContext } from './types.js';
-import { IdbQueryDb } from '../IdbQueryDb.js';
+import { PersistenceDocumentDb, PersistenceNamespace } from '../interfaces.js';
 
 function getMigrationMutations({
 	migration,
@@ -61,14 +61,12 @@ function getMigrationMutations({
 function getMigrationQueries({
 	migration,
 	context,
-	queryDb,
+	documents,
 }: {
 	migration: Migration<any>;
 	context: OpenDocumentDbContext;
-	queryDb: IDBDatabase;
+	documents: PersistenceDocumentDb;
 }) {
-	const queries = new IdbQueryDb(queryDb, context);
-
 	return migration.oldCollections.reduce((acc, collectionName) => {
 		acc[collectionName] = {
 			get: async (id: string) => {
@@ -80,7 +78,7 @@ function getMigrationQueries({
 				return doc;
 			},
 			findOne: async (filter: CollectionFilter) => {
-				const oid = await queries.findOneOid({
+				const oid = await documents.findOneOid({
 					collection: collectionName,
 					index: filter,
 				});
@@ -92,7 +90,7 @@ function getMigrationQueries({
 				return doc;
 			},
 			findAll: async (filter: CollectionFilter) => {
-				const { result: oids } = await queries.findAllOids({
+				const { result: oids } = await documents.findAllOids({
 					collection: collectionName,
 					index: filter,
 				});
@@ -111,27 +109,36 @@ function getMigrationQueries({
 	}, {} as any);
 }
 
-export function getMigrationEngine({
+export async function getMigrationEngine({
 	migration,
 	context,
-	queryDb,
+	ns,
 }: {
 	log?: (...args: any[]) => void;
 	migration: Migration;
 	context: OpenDocumentDbContext;
-	queryDb: IDBDatabase;
-}): MigrationEngine {
+	ns: PersistenceNamespace;
+}): Promise<MigrationEngine> {
+	const migrationContext = {
+		...context,
+		schema: migration.oldSchema,
+	};
+	if (migration.oldSchema.version === 0) {
+		return getInitialMigrationEngine({ migration, context: migrationContext });
+	}
+
 	const newOids = new Array<ObjectIdentifier>();
 
+	const documents = await ns.openDocuments(migrationContext);
 	const queries = getMigrationQueries({
 		migration,
-		context,
-		queryDb,
+		context: migrationContext,
+		documents,
 	});
 	const mutations = getMigrationMutations({
 		migration,
 		newOids,
-		ctx: context,
+		ctx: migrationContext,
 	});
 	const deleteCollection = async (collection: string) => {
 		await context.meta.deleteCollection(collection);
@@ -143,6 +150,10 @@ export function getMigrationEngine({
 		deleteCollection,
 		migrate: async (collection, strategy) => {
 			const docs = await queries[collection].findAll();
+			context.log(
+				'debug',
+				`Migrating ${docs.length} documents in ${collection}`,
+			);
 
 			await Promise.all(
 				docs.filter(Boolean).map(async (doc: any) => {
@@ -190,11 +201,14 @@ export function getMigrationEngine({
 		queries,
 		mutations,
 		awaitables,
+		close: async () => {
+			await documents.close();
+		},
 	};
 	return engine;
 }
 
-export function getInitialMigrationEngine({
+function getInitialMigrationEngine({
 	migration,
 	context,
 }: {
@@ -232,6 +246,7 @@ export function getInitialMigrationEngine({
 		queries,
 		mutations,
 		awaitables: [],
+		close: () => Promise.resolve(),
 	};
 	return engine;
 }

package/src/persistence/migration/finalize.ts

@@ -0,0 +1,126 @@
+import {
+	decomposeOid,
+	getOidRoot,
+	Migration,
+	MigrationEngine,
+} from '@verdant-web/common';
+import { ClientOperation, PersistenceDocumentDb } from '../interfaces.js';
+import { OpenDocumentDbContext } from './types.js';
+
+export async function finalizeMigration({
+	ctx,
+	documents,
+	migration,
+	engine,
+}: {
+	ctx: OpenDocumentDbContext;
+	documents: PersistenceDocumentDb;
+	migration: Migration<any>;
+	engine: MigrationEngine;
+}) {
+	/**
+	 * In cases where operations from the future have been
+	 * received by this client, we may have created entire
+	 * documents in metadata which were not written to storage
+	 * because all of their operations were in the future (
+	 * i.e. in the next version). We have to find those documents
+	 * and also write their snapshots to storage, because they
+	 * won't be present in storage already to 'refresh,' so
+	 * if we don't analyze metadata for 'future' operations like
+	 * this, we won't know they exist.
+	 *
+	 * This led to behavior where the metadata would be properly
+	 * synced, but after upgrading the app and migrating, items
+	 * would be missing from findAll and findOne queries.
+	 */
+	const docsWithUnappliedMigrations = await getDocsWithUnappliedMigrations({
+		currentVersion: migration.oldSchema.version,
+		newVersion: migration.newSchema.version,
+		ctx,
+	});
+
+	// once the schema is ready, we can write back the migrated documents
+
+	for (const collection of migration.allCollections) {
+		// map the keys to OIDs
+		const { result: oids } = await documents.findAllOids({
+			collection,
+		});
+		oids.push(
+			...engine.newOids.filter((oid) => {
+				return decomposeOid(oid).collection === collection;
+			}),
+			...docsWithUnappliedMigrations.filter((oid) => {
+				return decomposeOid(oid).collection === collection;
+			}),
+		);
+
+		const snapshots = await Promise.all(
+			oids.map(async (oid) => {
+				try {
+					const snap = await ctx.meta.getDocumentSnapshot(oid);
+					return [oid, snap];
+				} catch (e) {
+					// this seems to happen with baselines/ops which are not fully
+					// cleaned up after deletion?
+					ctx.log(
+						'error',
+						'Could not regenerate snapshot during migration for oid',
+						oid,
+						'this document will not be preserved',
+						e,
+					);
+					return null;
+				}
+			}),
+		);
+
+		const views: [string, any][] = snapshots.filter(
+			(s: any): s is [string, any] => !!s,
+		);
+
+		// now we can write the documents back
+		await documents.saveEntities(
+			views.map(([oid, snapshot]) => ({
+				oid,
+				getSnapshot() {
+					return snapshot;
+				},
+			})),
+			{
+				collections: [collection],
+			},
+		);
+	}
+}
+
+/**
+ * Gets a list of root OIDs for all documents which had operations stored already
+ * that were not applied to their queryable snapshots because they were in the
+ * future. These documents need to be refreshed in storage.
+ */
+async function getDocsWithUnappliedMigrations({
+	currentVersion,
+	newVersion: _,
+	ctx,
+}: {
+	currentVersion: number;
+	newVersion: number;
+	ctx: OpenDocumentDbContext;
+}) {
+	// scan for all operations in metadata after the current version.
+	// this could be more efficient if also filtering below or equal newVersion but
+	// that seems so unlikely in practice...
+	const unappliedOperations: ClientOperation[] = [];
+	await ctx.meta.iterateAllOperations(
+		(op) => {
+			unappliedOperations.push(op);
+		},
+		{
+			from: ctx.time.zeroWithVersion(currentVersion + 1),
+		},
+	);
+	return Array.from(
+		new Set(unappliedOperations.map((op) => getOidRoot(op.oid))),
+	);
+}

package/src/persistence/migration/migrate.ts

@@ -0,0 +1,169 @@
+import { Migration } from '@verdant-web/common';
+import { getMigrationPath } from './paths.js';
+import { OpenDocumentDbContext } from './types.js';
+import { getMigrationEngine } from './engine.js';
+import { finalizeMigration } from './finalize.js';
+import { PersistenceNamespace } from '../interfaces.js';
+import { ShutdownHandler } from '../../context/ShutdownHandler.js';
+
+export async function migrate({
+	context,
+	version,
+}: {
+	context: OpenDocumentDbContext;
+	version: number;
+}) {
+	const ns = await context.persistence.openNamespace(
+		context.namespace,
+		context,
+	);
+	await acquireLock(context.namespace, async () => {
+		const currentVersion = await context.persistence.getNamespaceVersion(
+			context.namespace,
+		);
+
+		context.log(
+			'debug',
+			'Opening index database',
+			context.namespace,
+			'Current database version:',
+			currentVersion,
+			'target version:',
+			version,
+			context.schema.wip ? '(wip)' : '',
+		);
+
+		const toRun = getMigrationPath({
+			currentVersion,
+			targetVersion: version,
+			migrations: context.migrations,
+		});
+
+		if (toRun.length > 0) {
+			context.log(
+				'debug',
+				'Migrations to run:',
+				toRun.map((m) => m.version),
+			);
+			await runMigrations({ context, ns, toRun });
+		}
+	});
+}
+
+async function acquireLock(namespace: string, procedure: () => Promise<void>) {
+	if (typeof navigator !== 'undefined' && navigator.locks) {
+		await navigator.locks.request(`verdant_migration_${namespace}`, procedure);
+	} else {
+		// TODO: is there a fallback?
+		await procedure();
+	}
+}
+
+export async function runMigrations({
+	context,
+	toRun,
+	ns,
+}: {
+	context: OpenDocumentDbContext;
+	toRun: Migration<any>[];
+	ns: PersistenceNamespace;
+}) {
+	// disable rebasing for the duration of migrations
+	context.pauseRebasing = true;
+	// now the fun part
+	for (const migration of toRun) {
+		context.log(
+			'info',
+			`🚀 Running migration v${migration.oldSchema.version} -> v${migration.newSchema.version}`,
+		);
+		const migrationContext = {
+			...context,
+			schema: migration.oldSchema,
+			shutdownHandler: new ShutdownHandler(),
+		};
+		// this will only write to our metadata store via operations!
+		const engine = await getMigrationEngine({
+			migration,
+			context: migrationContext,
+			ns,
+		});
+		try {
+			context.log(
+				'debug',
+				'Migrating data',
+				migrationContext.namespace,
+				'from version',
+				migration.oldSchema.version,
+				'to version',
+				migration.newSchema.version,
+			);
+			await migration.migrate(engine);
+			// wait on any out-of-band async operations to complete
+			await Promise.all(engine.awaitables);
+		} catch (err) {
+			context.log(
+				'critical',
+				`Migration failed (${migration.oldSchema.version} -> ${migration.newSchema.version})`,
+				err,
+			);
+			if (err instanceof Error) {
+				throw err;
+			} else {
+				throw new Error('Unknown error during migration');
+			}
+		}
+
+		await engine.close();
+
+		migrationContext.log(
+			'debug',
+			'Upgrading database',
+			migrationContext.namespace,
+			'from version',
+			migrationContext.schema.version,
+			'to version',
+			migration.newSchema.version,
+		);
+
+		await ns.applyMigration(migrationContext, migration);
+
+		// switch to the new schema
+		migrationContext.schema = migration.newSchema;
+		const upgradedDocuments = await ns.openDocuments(migrationContext);
+
+		await finalizeMigration({
+			ctx: migrationContext,
+			migration,
+			engine,
+			documents: upgradedDocuments,
+		});
+		await upgradedDocuments.close();
+
+		migrationContext.log(
+			'debug',
+			`Migration of ${migrationContext.namespace} complete.`,
+		);
+		migrationContext.log(
+			'info',
+			`
+				⬆️ v${migration.newSchema.version} Migration complete. Here's the rundown:
+					- Added collections: ${migration.addedCollections.join(', ')}
+					- Removed collections: ${migration.removedCollections.join(', ')}
+					- Changed collections: ${migration.changedCollections.join(', ')}
+					- New indexes: ${Object.keys(migration.addedIndexes)
+						.map((col) =>
+							migration.addedIndexes[col].map((i) => `${col}.${i.name}`),
+						)
+						.flatMap((i) => i)
+						.join(', ')}
+					- Removed indexes: ${Object.keys(migration.removedIndexes)
+						.map((col) =>
+							migration.removedIndexes[col].map((i) => `${col}.${i.name}`),
+						)
+						.flatMap((i) => i)
+						.join(', ')}
+			`,
+		);
+	}
+	context.pauseRebasing = false;
+}

package/src/persistence/migration/types.ts

@@ -0,0 +1,4 @@
+import { Context } from '../../context/context.js';
+
+/** During migration, only a partial context is available */
+export type OpenDocumentDbContext = Omit<Context, 'documents' | 'files'>;

package/src/persistence/persistence.ts

@@ -4,7 +4,9 @@ import { getWipNamespace } from '../utils/wip.js';
 import { ExportedData } from './interfaces.js';
 import { PersistenceFiles } from './PersistenceFiles.js';
 import { PersistenceMetadata } from './PersistenceMetadata.js';
-import { PersistenceQueries } from './PersistenceQueries.js';
+import { PersistenceDocuments } from './PersistenceQueries.js';
+import { migrate } from './migration/migrate.js';
+import { ShutdownHandler } from '../context/ShutdownHandler.js';
 
 export async function initializePersistence(
 	ctx: InitialContext,
@@ -27,6 +29,7 @@ export async function initializePersistence(
 
 			if (currentVersion === 0) {
 				// there is no existing data. nothing to copy.
+				context.log('debug', 'No existing data to copy to WIP namespace');
 			} else {
 				const currentSchema = ctx.oldSchemas?.find(
 					(s) => s.version === currentVersion,
@@ -45,6 +48,9 @@ export async function initializePersistence(
 				await context.persistence.copyNamespace(
 					context.originalNamespace,
 					context.namespace,
+					// needs to be the original schema; the copy should be of the original
+					// data and schema structure; the WIP schema migration application happens
+					// below.
 					{
 						...context,
 						schema: currentSchema,
@@ -54,21 +60,32 @@ export async function initializePersistence(
 		}
 	}
 
+	const namespace = await ctx.persistence.openNamespace(
+		context.namespace,
+		context,
+	);
+
 	context.log('info', 'Opening persistence metadata');
 	context.meta = new PersistenceMetadata(
-		await ctx.persistence.openMetadata(ctx),
+		await namespace.openMetadata(ctx),
 		ctx,
 	);
 
 	context.log('info', 'Opening persistence files');
 	context.files = new PersistenceFiles(
-		await ctx.persistence.openFiles(context),
+		await namespace.openFiles(context),
 		context,
 	);
 
-	context.log('info', 'Opening persistence queries');
-	context.queries = new PersistenceQueries(
-		await ctx.persistence.openQueries(context),
+	context.log('info', 'Migrating document database');
+	await migrate({
+		context,
+		version: ctx.schema.version,
+	});
+
+	context.log('info', 'Opening persistence documents');
+	context.documents = new PersistenceDocuments(
+		await namespace.openDocuments(context),
 		context,
 	);
 
@@ -120,6 +137,7 @@ export async function importPersistence(
 				disableRebasing: true,
 			},
 		},
+		persistenceShutdownHandler: new ShutdownHandler(),
 	});
 	// load imported data into persistence
 	await importedContext.meta.resetFrom(exportedData.data);
@@ -140,40 +158,42 @@ export async function importPersistence(
 			};
 		}),
 	);
-	await importedContext.queries.saveEntities(toSave);
+	await importedContext.documents.saveEntities(toSave);
 	await importedContext.files.import(exportedData);
 
 	ctx.log('debug', 'Imported data into temporary namespace', importedNamespace);
 
 	// shut down the imported databases
-	await importedContext.queries.dispose();
-	await importedContext.meta.dispose();
-	await importedContext.files.dispose();
+	await importedContext.persistenceShutdownHandler.shutdown();
 
 	if (exportedSchema.version !== ctx.schema.version) {
 		// an upgrade of the imported data is needed ; it's an older version
 		// of the schema.
-		ctx.log('debug', 'Shut down imported databases');
 
 		// upgrade the imported data to the latest schema
 		const currentSchema = ctx.schema;
 		const upgradedContext = await initializePersistence({
 			...importedContext,
+			persistenceShutdownHandler: new ShutdownHandler(),
 			schema: currentSchema,
 		});
 
 		ctx.log('debug', 'Upgraded imported data to current schema');
 
-		await upgradedContext.queries.dispose();
-		await upgradedContext.meta.dispose();
-		await upgradedContext.files.dispose();
+		await upgradedContext.persistenceShutdownHandler.shutdown();
 
 		ctx.log('debug', 'Shut down upgraded databases');
 	}
 
+	// shut down the persistence layer
+	await ctx.persistenceShutdownHandler.shutdown();
+
 	// copy the imported data into the current namespace
 	await ctx.persistence.copyNamespace(importedNamespace, ctx.namespace, ctx);
 
+	// restart the persistence layer
+	await initializePersistence(ctx);
+
 	// verify integrity -- this can only be done if imported data was same
 	// version as current schema, because migrations could add or remove
 	// operations. still, it's a good sanity check.
@@ -220,4 +240,7 @@ export async function importPersistence(
 
 	ctx.internalEvents.emit('persistenceReset');
 	ctx.log('info', 'Data imported successfully');
+
+	// reset to allow future shutdowns.
+	ctx.persistenceShutdownHandler.reset();
 }

package/src/queries/FindAllQuery.ts

@@ -23,7 +23,7 @@ export class FindAllQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result: oids } = await this.context.queries.findAllOids({
+		const { result: oids } = await this.context.documents.findAllOids({
 			collection: this.collection,
 			index: this.index,
 		});

package/src/queries/FindInfiniteQuery.ts

@@ -37,7 +37,7 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await this.context.queries.findAllOids({
+		const { result, hasNextPage } = await this.context.documents.findAllOids({
 			collection: this.collection,
 			limit: this._pageSize * this._upToPage,
 			offset: 0,
@@ -48,7 +48,7 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	};
 
 	public loadMore = async () => {
-		const { result, hasNextPage } = await this.context.queries.findAllOids({
+		const { result, hasNextPage } = await this.context.documents.findAllOids({
 			collection: this.collection,
 			limit: this._pageSize,
 			offset: this._pageSize * this._upToPage,

package/src/queries/FindOneQuery.ts

@@ -23,7 +23,7 @@ export class FindOneQuery<T> extends BaseQuery<T | null> {
 	}
 
 	protected run = async () => {
-		const oid = await this.context.queries.findOneOid({
+		const oid = await this.context.documents.findOneOid({
 			collection: this.collection,
 			index: this.index,
 		});

package/src/queries/FindPageQuery.ts

@@ -48,7 +48,7 @@ export class FindPageQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await this.context.queries.findAllOids({
+		const { result, hasNextPage } = await this.context.documents.findAllOids({
 			collection: this.collection,
 			index: this.index,
 			limit: this._pageSize,

package/src/sync/FileSync.ts

@@ -42,7 +42,7 @@ export class FileSync extends Disposable {
 		this.ctx.log('debug', 'Uploading file', data.id, data.name);
 		try {
 			await this.uploadFile(data);
-			this.ctx.internalEvents.emit(`fileUploaded:${data.id}`);
+			this.ctx.internalEvents.emit(`fileUploaded:${data.id}`, data);
 		} catch (e) {
 			this.ctx.log('error', 'File upload failed', e);
 		}
@@ -70,14 +70,17 @@ export class FileSync extends Disposable {
 		formData.append('file', file);
 
 		try {
-			const response = await fetch(fileEndpoint + `/${data.id}`, {
-				method: 'POST',
-				body: formData,
-				credentials: 'include',
-				headers: {
-					Authorization: `Bearer ${token}`,
+			const response = await this.ctx.environment.fetch(
+				fileEndpoint + `/${data.id}`,
+				{
+					method: 'POST',
+					body: formData,
+					credentials: 'include',
+					headers: {
+						Authorization: `Bearer ${token}`,
+					},
 				},
-			});
+			);
 
 			if (response.ok) {
 				this.ctx.log('info', 'File upload successful');
@@ -132,14 +135,17 @@ export class FileSync extends Disposable {
 			await this.endpointProvider.getEndpoints();
 
 		try {
-			const response = await fetch(fileEndpoint + `/${id}`, {
-				method: 'GET',
-				credentials: 'include',
-				headers: {
-					'Content-Type': 'application/json',
-					Authorization: `Bearer ${token}`,
+			const response = await this.ctx.environment.fetch(
+				fileEndpoint + `/${id}`,
+				{
+					method: 'GET',
+					credentials: 'include',
+					headers: {
+						'Content-Type': 'application/json',
+						Authorization: `Bearer ${token}`,
+					},
 				},
-			});
+			);
 
 			if (response.ok) {
 				const data = await response.json();

package/src/sync/PushPullSync.ts

@@ -20,7 +20,9 @@ export class PushPullSync
 	readonly presence: PresenceManager;
 	private endpointProvider;
 	private heartbeat;
-	private fetch;
+	private get fetch() {
+		return this.ctx.environment.fetch;
+	}
 
 	readonly mode = 'pull';
 	private ctx;
@@ -33,20 +35,17 @@ export class PushPullSync
 		endpointProvider,
 		presence,
 		interval = 15 * 1000,
-		fetch = window.fetch.bind(window),
 		ctx,
 	}: {
 		endpointProvider: ServerSyncEndpointProvider;
 		presence: PresenceManager;
 		interval?: number;
-		fetch?: typeof window.fetch;
 		ctx: Context;
 	}) {
 		super();
 		this.ctx = ctx;
 		this.presence = presence;
 		this.endpointProvider = endpointProvider;
-		this.fetch = fetch;
 
 		this.heartbeat = new Heartbeat({
 			interval,