@verdant-web/store 3.12.0 → 4.0.0-next.0

package/src/persistence/idb/queries/IdbQueryDb.ts

@@ -0,0 +1,251 @@
+import {
+	CollectionFilter,
+	createOid,
+	decomposeOid,
+	getIndexValues,
+	ObjectIdentifier,
+} from '@verdant-web/common';
+import { Context } from '../../../context/context.js';
+import {
+	AbstractTransaction,
+	CommonQueryOptions,
+	PersistenceQueryDb,
+	QueryMode,
+} from '../../interfaces.js';
+import { IdbService } from '../IdbService.js';
+import { getRange } from './ranges.js';
+import { closeDatabase, getSizeOfObjectStore, isAbortError } from '../util.js';
+
+export class IdbQueryDb extends IdbService implements PersistenceQueryDb {
+	private ctx;
+	constructor(db: IDBDatabase, context: Omit<Context, 'queries' | 'files'>) {
+		super(db, { log: context.log });
+		this.ctx = context;
+	}
+
+	stats = async (): Promise<
+		Record<string, { count: number; size: number }>
+	> => {
+		const collectionNames = Object.keys(this.ctx.schema.collections);
+		const collections: Record<string, { count: number; size: number }> = {};
+		await Promise.all(
+			collectionNames.map(async (name) => {
+				const size = await getSizeOfObjectStore(this.db, name);
+				collections[name] = size;
+			}),
+		);
+		return collections;
+	};
+
+	transaction = (opts: {
+		mode?: QueryMode;
+		storeNames: string[];
+		abort?: AbortSignal;
+	}): AbstractTransaction => {
+		return this.createTransaction(opts.storeNames, {
+			mode: opts.mode,
+			abort: opts.abort,
+		});
+	};
+
+	findOneOid = async (opts: {
+		collection: string;
+		index?: CollectionFilter;
+	}): Promise<ObjectIdentifier | null> => {
+		const result = await this.run<IDBCursorWithValue | null>(
+			opts.collection,
+			(store) => {
+				const source = opts.index?.where
+					? store.index(opts.index.where)
+					: store;
+				const direction = opts.index?.order === 'desc' ? 'prev' : 'next';
+				const range = getRange(this.ctx.schema, opts.collection, opts.index);
+				return source.openCursor(range, direction);
+			},
+			{ mode: 'readonly' },
+		);
+		if (result) {
+			return createOid(opts.collection, result.primaryKey.toString());
+		}
+		return null;
+	};
+	findAllOids = async ({
+		collection,
+		index,
+		offset,
+		limit,
+	}: {
+		collection: string;
+		index?: CollectionFilter;
+		limit?: number;
+		offset?: number;
+	}): Promise<{ result: ObjectIdentifier[]; hasNextPage: boolean }> => {
+		const tx = this.createTransaction([collection], { mode: 'readonly' });
+		const store = tx.objectStore(collection);
+		const source = index?.where ? store.index(index.where) : store;
+		const direction = index?.order === 'desc' ? 'prev' : 'next';
+		const range = getRange(this.ctx.schema, collection, index);
+		const request = source.openCursor(range, direction);
+
+		let hasNextPage = false;
+		const result = await new Promise<string[]>((resolve, reject) => {
+			let hasDoneOffset = !offset;
+			let visited = 0;
+			const results = new Set<ObjectIdentifier>();
+
+			request.onsuccess = () => {
+				visited++;
+				const cursor = request.result as IDBCursorWithValue | null;
+				if (!cursor) {
+					resolve(Array.from(results));
+					return;
+				}
+
+				// first offset, if we have one. cursor opens at beginning.
+				if (offset && !hasDoneOffset) {
+					cursor.advance(offset);
+					hasDoneOffset = true;
+					// next iteration we begin adding results.
+				} else {
+					// add result to set, unless we have reached limit.
+					if (!limit || results.size < limit) {
+						results.add(createOid(collection, cursor.primaryKey.toString()));
+					}
+					// even if we reached limit, we keep going one more to check if there's
+					// a next page.
+					if (limit && visited > limit) {
+						hasNextPage = true;
+						// stop iteration here; we reached limit and we have next page
+						// info we need.
+						resolve(Array.from(results));
+					} else {
+						cursor.continue();
+					}
+				}
+			};
+
+			request.onerror = () => {
+				if (request.error?.name === 'InvalidStateError') {
+					this.ctx.log(
+						'error',
+						`find query failed with InvalidStateError`,
+						request.error,
+					);
+					resolve([]);
+				} else if (request.error && isAbortError(request.error)) {
+					resolve([]);
+				} else {
+					reject(request.error);
+				}
+			};
+		});
+
+		return {
+			result,
+			hasNextPage,
+		};
+	};
+
+	saveEntities = async (
+		entities: { oid: ObjectIdentifier; getSnapshot: () => any }[],
+		opts?: CommonQueryOptions & { abort?: AbortSignal },
+	): Promise<void> => {
+		if (entities.length === 0) return;
+
+		let collections = Array.from(
+			new Set(entities.map((e) => decomposeOid(e.oid).collection)),
+		);
+
+		const toRemove = collections.filter((c) => !this.ctx.schema.collections[c]);
+		if (toRemove.length > 0) {
+			this.ctx.log(
+				'warn',
+				`Ignoring entities from collections that no longer exist: ${toRemove.join(
+					', ',
+				)}`,
+			);
+		}
+		const withRemoved = new Set(collections);
+		toRemove.forEach((c) => withRemoved.delete(c));
+		collections = Array.from(withRemoved);
+
+		const options = {
+			transaction: this.createTransaction(collections, {
+				mode: 'readwrite',
+				abort: opts?.abort,
+			}),
+		};
+
+		// FIXME: not test is making it to this line
+
+		await Promise.all(
+			entities.map(async (e) => {
+				const snapshot = e.getSnapshot();
+				try {
+					await this.saveDocument(e.oid, snapshot, options);
+				} catch (err) {
+					this.ctx.log(
+						'error',
+						`Error saving document ${e.oid} (${JSON.stringify(snapshot)})`,
+						err,
+					);
+					if (err instanceof Error) {
+						throw err;
+					} else {
+						throw new Error('Unknown error saving document');
+					}
+				}
+			}),
+		);
+		options.transaction.commit();
+		this.ctx.entityEvents.emit('collectionsChanged', collections);
+		for (const entity of entities) {
+			this.ctx.entityEvents.emit('documentChanged', entity.oid);
+		}
+	};
+
+	reset = async (opts?: {
+		transaction?: AbstractTransaction;
+	}): Promise<void> => {
+		const names = Object.keys(this.ctx.schema.collections);
+		const tx =
+			(opts?.transaction as IDBTransaction) ||
+			this.createTransaction(names, { mode: 'readwrite' });
+		await Promise.all(
+			names.map((name) =>
+				this.run(name, (store) => store.clear(), { transaction: tx }),
+			),
+		);
+		this.ctx.entityEvents.emit('collectionsChanged', names);
+		this.ctx.log('info', '💨 Reset queryable storage');
+	};
+
+	dispose = () => {
+		return closeDatabase(this.db);
+	};
+
+	private saveDocument = async (
+		oid: ObjectIdentifier,
+		doc: any,
+		{ transaction }: { transaction?: IDBTransaction },
+	) => {
+		this.ctx.log('debug', `Saving document indexes for querying ${oid}`);
+		const { collection, id } = decomposeOid(oid);
+		if (!doc) {
+			await this.run(collection, (store) => store.delete(id), {
+				mode: 'readwrite',
+				transaction,
+			});
+			this.ctx.log('debug', `Deleted document indexes for querying ${oid}`);
+		} else {
+			const schema = this.ctx.schema.collections[collection];
+			// no need to validate before storing; the entity's snapshot is already validated.
+			const indexes = getIndexValues(schema, doc);
+			await this.run(collection, (store) => store.put(indexes), {
+				mode: 'readwrite',
+				transaction,
+			});
+			this.ctx.log('debug', `Save complete for ${oid}`, indexes);
+		}
+	};
+}

package/src/{migration → persistence/idb/queries/migration}/db.ts

@@ -1,48 +1,23 @@
-import { closeDatabase, globalIDB, storeRequestPromise } from '../idb.js';
+import {
+	globalIDB,
+	storeRequestPromise,
+	openDatabase as baseOpenDatabase,
+	getDocumentDbName,
+} from '../../util.js';
 import { OpenDocumentDbContext } from './types.js';
 
 export async function getDatabaseVersion(
 	indexedDB: IDBFactory,
 	namespace: string,
-	version: number,
-	log?: (...args: any[]) => void,
 ): Promise<number> {
-	function openAndGetVersion(
-		resolve: (res: [number, IDBDatabase]) => void,
-		reject: (err: Error) => void,
-	) {
-		let currentVersion: number;
-		let database: IDBDatabase;
-		const request = indexedDB.open(
-			[namespace, 'collections'].join('_'),
-			version,
-		);
-		request.onupgradeneeded = async (event) => {
-			currentVersion = event.oldVersion;
-			const transaction = request.transaction!;
-			database = request.result;
-			transaction.abort();
-		};
-		request.onsuccess = (event) => {
-			resolve([request.result.version, request.result]);
-		};
-		request.onblocked = (event) => {
-			// retry if blocked
-			log?.('Database blocked, waiting...');
-			// setTimeout(() => {
-			// 	openAndGetVersion(resolve, reject);
-			// }, 200);
-		};
-		request.onerror = (event) => {
-			// FIXME: this fails if the code is older than the local database
-			resolve([currentVersion!, database!]);
-		};
+	const databaseName = getDocumentDbName(namespace);
+	const dbInfo = await indexedDB.databases();
+	const existingDb = dbInfo.find((info) => info.name === databaseName);
+	if (existingDb) {
+		return existingDb.version ?? 0;
 	}
-	const [currentVersion, db] = await new Promise<[number, IDBDatabase]>(
-		openAndGetVersion,
-	);
-	await closeDatabase(db);
-	return currentVersion;
+
+	return 0;
 }
 
 /**
@@ -121,40 +96,11 @@ export async function openDatabase({
 	context: OpenDocumentDbContext;
 }): Promise<IDBDatabase> {
 	context.log('debug', 'Opening database', namespace, 'at version', version);
-	const db = await new Promise<IDBDatabase>((resolve, reject) => {
-		const request = indexedDB.open(
-			[namespace, 'collections'].join('_'),
-			version,
-		);
-		request.onupgradeneeded = async (event) => {
-			const transaction = request.transaction!;
-			transaction.abort();
-
-			context.log(
-				'error',
-				'Database upgrade needed, but not expected',
-				'Expected',
-				version,
-				'Got',
-				request.result.version,
-			);
-			reject(
-				request.error ||
-					new Error(
-						`Migration error: database version changed unexpectedly when reading current data. Expected ${version}, got ${request.result.version}`,
-					),
-			);
-		};
-		request.onsuccess = (event) => {
-			resolve(request.result);
-		};
-		request.onblocked = (event) => {
-			reject(new Error('Migration error: database blocked'));
-		};
-		request.onerror = (event) => {
-			reject(new Error('Migration error: database error'));
-		};
-	});
+	const db = await baseOpenDatabase(
+		getDocumentDbName(namespace),
+		version,
+		indexedDB,
+	);
 
 	db.addEventListener('versionchange', (event) => {
 		db.close();

package/src/{migration → persistence/idb/queries/migration}/engine.ts

@@ -10,25 +10,20 @@ import {
 	createOid,
 	diffToPatches,
 	getOid,
-	initialToPatches,
 	removeOidPropertiesFromAllSubObjects,
 	AuthorizationKey,
 } from '@verdant-web/common';
-import { Context } from '../context.js';
-import { Metadata } from '../metadata/Metadata.js';
-import { findAllOids, findOneOid } from '../queries/dbQueries.js';
 import { OpenDocumentDbContext } from './types.js';
+import { IdbQueryDb } from '../IdbQueryDb.js';
 
 function getMigrationMutations({
 	migration,
-	meta,
-	getMigrationNow,
 	newOids,
+	ctx,
 }: {
 	migration: Migration<any>;
 	newOids: string[];
-	getMigrationNow: () => string;
-	meta: Metadata;
+	ctx: OpenDocumentDbContext;
 }) {
 	return migration.allCollections.reduce((acc, collectionName) => {
 		acc[collectionName] = {
@@ -40,24 +35,22 @@ function getMigrationMutations({
 				const oid = createOid(collectionName, primaryKey);
 				newOids.push(oid);
 
-				await meta.insertLocalOperations(
-					initialToPatches(doc, oid, getMigrationNow, undefined, undefined, {
-						authz: options?.access,
+				await ctx.time.withMigrationTime(migration.version, () =>
+					ctx.meta.insertData({
+						operations: ctx.patchCreator.createInitialize(
+							doc,
+							oid,
+							options?.access,
+						),
+						isLocal: true,
 					}),
 				);
 				return doc;
 			},
 			delete: async (id: string) => {
 				const rootOid = createOid(collectionName, id);
-				const authz = await meta.getDocumentAuthz(rootOid);
-				const allOids = await meta.getAllDocumentRelatedOids(rootOid);
-				return meta.insertLocalOperations(
-					allOids.map((oid) => ({
-						oid,
-						timestamp: getMigrationNow(),
-						data: { op: 'delete' },
-						authz,
-					})),
+				await ctx.time.withMigrationTime(migration.version, () =>
+					ctx.meta.deleteDocument(rootOid),
 				);
 			},
 		};
@@ -68,46 +61,46 @@ function getMigrationMutations({
 function getMigrationQueries({
 	migration,
 	context,
-	meta,
+	queryDb,
 }: {
 	migration: Migration<any>;
-	context: Context;
-	meta: Metadata;
+	context: OpenDocumentDbContext;
+	queryDb: IDBDatabase;
 }) {
+	const queries = new IdbQueryDb(queryDb, context);
+
 	return migration.oldCollections.reduce((acc, collectionName) => {
 		acc[collectionName] = {
 			get: async (id: string) => {
 				const oid = createOid(collectionName, id);
-				const doc = await meta.getDocumentSnapshot(oid, {
+				const doc = await context.meta.getDocumentSnapshot(oid, {
 					// only get the snapshot up to the previous version (newer operations may have synced)
-					to: meta.time.now(migration.oldSchema.version),
+					to: context.time.nowWithVersion(migration.oldSchema.version),
 				});
 				return doc;
 			},
 			findOne: async (filter: CollectionFilter) => {
-				const oid = await findOneOid({
+				const oid = await queries.findOneOid({
 					collection: collectionName,
 					index: filter,
-					context,
 				});
 				if (!oid) return null;
-				const doc = await meta.getDocumentSnapshot(oid, {
+				const doc = await context.meta.getDocumentSnapshot(oid, {
 					// only get the snapshot up to the previous version (newer operations may have synced)
-					to: meta.time.now(migration.oldSchema.version),
+					to: context.time.nowWithVersion(migration.oldSchema.version),
 				});
 				return doc;
 			},
 			findAll: async (filter: CollectionFilter) => {
-				const oids = await findAllOids({
+				const { result: oids } = await queries.findAllOids({
 					collection: collectionName,
 					index: filter,
-					context,
 				});
 				const docs = await Promise.all(
 					oids.map((oid) =>
-						meta.getDocumentSnapshot(oid, {
+						context.meta.getDocumentSnapshot(oid, {
 							// only get the snapshot up to the previous version (newer operations may have synced)
-							to: meta.time.now(migration.oldSchema.version),
+							to: context.time.nowWithVersion(migration.oldSchema.version),
 						}),
 					),
 				);
@@ -119,41 +112,29 @@ function getMigrationQueries({
 }
 
 export function getMigrationEngine({
-	meta,
 	migration,
 	context,
+	queryDb,
 }: {
 	log?: (...args: any[]) => void;
 	migration: Migration;
-	meta: Metadata;
-	context: Context;
+	context: OpenDocumentDbContext;
+	queryDb: IDBDatabase;
 }): MigrationEngine {
-	function getMigrationNow() {
-		return meta.time.zero(migration.version);
-	}
-
 	const newOids = new Array<ObjectIdentifier>();
 
 	const queries = getMigrationQueries({
 		migration,
 		context,
-		meta,
+		queryDb,
 	});
 	const mutations = getMigrationMutations({
 		migration,
-		getMigrationNow,
 		newOids,
-		meta,
+		ctx: context,
 	});
 	const deleteCollection = async (collection: string) => {
-		const allOids = await meta.getAllCollectionRelatedOids(collection);
-		return meta.insertLocalOperations(
-			allOids.map((oid) => ({
-				oid,
-				timestamp: getMigrationNow(),
-				data: { op: 'delete' },
-			})),
-		);
+		await context.meta.deleteCollection(collection);
 	};
 	const awaitables = new Array<Promise<any>>();
 	const engine: MigrationEngine = {
@@ -174,7 +155,7 @@ export function getMigrationEngine({
 					// when the snapshots themselves are derived from the same data...)
 					// maybe don't use the findAll query, and instead go a level
 					// lower to retain access to lower level data here?
-					const authz = await meta.getDocumentAuthz(rootOid);
+					const authz = await context.meta.getDocumentAuthz(rootOid);
 					const original = cloneDeep(doc);
 					// @ts-ignore - excessive type resolution
 					const newValue = await strategy(doc);
@@ -188,7 +169,7 @@ export function getMigrationEngine({
 						const patches = diffToPatches(
 							original,
 							newValue,
-							getMigrationNow,
+							() => context.time.zeroWithVersion(migration.version),
 							undefined,
 							[],
 							{
@@ -197,7 +178,10 @@ export function getMigrationEngine({
 							},
 						);
 						if (patches.length > 0) {
-							await meta.insertLocalOperations(patches);
+							await context.meta.insertData({
+								operations: patches,
+								isLocal: true,
+							});
 						}
 					}
 				}),
@@ -211,18 +195,12 @@ export function getMigrationEngine({
 }
 
 export function getInitialMigrationEngine({
-	meta,
 	migration,
 	context,
 }: {
 	context: OpenDocumentDbContext;
 	migration: Migration;
-	meta: Metadata;
 }): MigrationEngine {
-	function getMigrationNow() {
-		return meta.time.zero(migration.version);
-	}
-
 	const newOids = new Array<ObjectIdentifier>();
 
 	const queries = new Proxy({} as any, {
@@ -235,9 +213,8 @@ export function getInitialMigrationEngine({
 
 	const mutations = getMigrationMutations({
 		migration,
-		getMigrationNow,
 		newOids,
-		meta,
+		ctx: context,
 	});
 	const engine: MigrationEngine = {
 		log: context.log,

package/src/{migration → persistence/idb/queries/migration}/migrations.ts

@@ -6,12 +6,11 @@ import {
 	getIndexValues,
 	getOidRoot,
 } from '@verdant-web/common';
-import { Metadata } from '../metadata/Metadata.js';
-import { ClientOperation } from '../metadata/OperationsStore.js';
+import { ClientOperation } from '../../../interfaces.js';
 import { acquireLock, openDatabase, upgradeDatabase } from './db.js';
 import { getInitialMigrationEngine, getMigrationEngine } from './engine.js';
 import { OpenDocumentDbContext } from './types.js';
-import { closeDatabase } from '../idb.js';
+import { closeDatabase } from '../../util.js';
 
 const globalIDB =
 	typeof window !== 'undefined' ? window.indexedDB : (undefined as any);
@@ -19,13 +18,11 @@ const globalIDB =
 export async function runMigrations({
 	context,
 	toRun,
-	meta,
 	indexedDB = globalIDB,
 	namespace = context.namespace,
 }: {
 	context: OpenDocumentDbContext;
 	toRun: Migration<any>[];
-	meta: Metadata;
 	indexedDB?: IDBFactory;
 	/** This namespace value controls where the database being migrated is. */
 	namespace?: string;
@@ -39,7 +36,6 @@ export async function runMigrations({
 			// migrations from 0 (i.e. initial migrations) don't attempt to open an existing db
 			if (migration.oldSchema.version === 0) {
 				engine = getInitialMigrationEngine({
-					meta,
 					migration,
 					context,
 				});
@@ -56,12 +52,9 @@ export async function runMigrations({
 
 				// this will only write to our metadata store via operations!
 				engine = getMigrationEngine({
-					meta,
 					migration,
-					context: {
-						...context,
-						documentDb: originalDatabase,
-					},
+					context,
+					queryDb: originalDatabase,
 				});
 				try {
 					await migration.migrate(engine);
@@ -115,9 +108,9 @@ export async function runMigrations({
 			 * would be missing from findAll and findOne queries.
 			 */
 			const docsWithUnappliedMigrations = await getDocsWithUnappliedMigrations({
-				meta,
 				currentVersion: migration.oldSchema.version,
 				newVersion: migration.newSchema.version,
+				ctx: context,
 			});
 
 			// once the schema is ready, we can write back the migrated documents
@@ -163,7 +156,7 @@ export async function runMigrations({
 				const snapshots = await Promise.all(
 					oids.map(async (oid) => {
 						try {
-							const snap = await meta.getDocumentSnapshot(oid);
+							const snap = await context.meta.getDocumentSnapshot(oid);
 							return [oid, snap];
 						} catch (e) {
 							// this seems to happen with baselines/ops which are not fully
@@ -279,22 +272,24 @@ async function putView(store: IDBObjectStore, view: any) {
  * future. These documents need to be refreshed in storage.
  */
 async function getDocsWithUnappliedMigrations({
-	meta,
 	currentVersion,
 	newVersion: _,
+	ctx,
 }: {
 	currentVersion: number;
 	newVersion: number;
-	meta: Metadata;
+	ctx: OpenDocumentDbContext;
 }) {
 	// scan for all operations in metadata after the current version.
 	// this could be more efficient if also filtering below or equal newVersion but
 	// that seems so unlikely in practice...
 	const unappliedOperations: ClientOperation[] = [];
-	await meta.operations.iterateOverAllOperations(
-		(op) => unappliedOperations.push(op),
+	await ctx.meta.iterateAllOperations(
+		(op) => {
+			unappliedOperations.push(op);
+		},
 		{
-			from: meta.time.zero(currentVersion + 1),
+			from: ctx.time.zeroWithVersion(currentVersion + 1),
 		},
 	);
 	return Array.from(