@verdant-web/store 2.5.8 → 2.7.0

package/src/entities/EntityStore.ts

@@ -9,6 +9,7 @@ import {
 	DocumentBaseline,
 	EventSubscriber,
 	generateId,
+	getIndexValues,
 	getOidRoot,
 	getUndoOperations,
 	groupBaselinesByRootOid,
@@ -18,6 +19,7 @@ import {
 	Operation,
 	removeOidsFromAllSubObjects,
 	StorageCollectionSchema,
+	StorageObjectFieldSchema,
 } from '@verdant-web/common';
 import { Context } from '../context.js';
 import { FileManager } from '../files/FileManager.js';
@@ -98,24 +100,37 @@ export class EntityStore {
 		this.context = context;
 	};
 
-	private getDocumentSchema = (oid: ObjectIdentifier) => {
+	private getDocumentSchema = (
+		oid: ObjectIdentifier,
+	): { schema: StorageObjectFieldSchema | null; readonlyKeys: string[] } => {
 		const { collection } = decomposeOid(oid);
 		if (!this.schema.collections[collection]) {
 			this.log('warn', `Missing schema for collection: ${collection}`);
-			return null;
+			return { schema: null, readonlyKeys: [] };
 		}
+		const schema = this.schema.collections[collection];
 		return {
-			type: 'object',
-			properties: this.schema.collections[collection].fields as any,
-		} as const;
+			readonlyKeys: [schema.primaryKey],
+			schema: {
+				type: 'object',
+				properties: schema.fields as any,
+			} as const,
+		};
 	};
 
 	private refreshFamilyCache = async (
 		familyCache: DocumentFamilyCache,
 		dropUnconfirmed = false,
+		dropAll = false,
 	) => {
 		// avoid writing to disposed db
-		if (this._disposed) return;
+		if (this._disposed) {
+			this.context.log(
+				'debug',
+				`EntityStore is disposed, not refreshing ${familyCache.oid} cache`,
+			);
+			return;
+		}
 
 		// metadata must be loaded from database to initialize family cache
 		const transaction = this.meta.createTransaction([
@@ -146,21 +161,40 @@ export class EntityStore {
 				{ transaction, mode: 'readwrite' },
 			),
 		]);
-		familyCache.reset(operations, baselines, dropUnconfirmed);
+		familyCache.reset({
+			operations,
+			baselines,
+			dropExistingUnconfirmed: dropUnconfirmed,
+			dropAll,
+		});
 	};
 
 	private openFamilyCache = async (oid: ObjectIdentifier) => {
 		const documentOid = getOidRoot(oid);
 		let familyCache = this.documentFamilyCaches.get(documentOid);
 		if (!familyCache) {
+			this.context.log('debug', 'opening family cache for', documentOid);
 			// metadata must be loaded from database to initialize family cache
 			familyCache = new DocumentFamilyCache({
 				oid: documentOid,
 				store: this,
 				context: this.context,
 			});
+
+			// PROBLEM: because the next line is async, it yields to
+			// queued promises which may need data from this cache,
+			// but the cache is empty. But if we move the set to
+			// after the async, we can clobber an existing cache
+			// with race conditions...
+			// So as an attempt to fix that, I've added a promise
+			// on DocumentFamilyCache which I manually resolve
+			// with setInitialized, then await initializedPromise
+			// further down even if there was a cache hit.
+			// Surely there is a better pattern for this.
+			// FIXME:
 			this.documentFamilyCaches.set(documentOid, familyCache);
 			await this.refreshFamilyCache(familyCache);
+			familyCache.setInitialized();
 
 			// this.unsubscribes.push(
 			// 	familyCache.subscribe('change:*', this.onEntityChange),
@@ -168,6 +202,7 @@ export class EntityStore {
 
 			// TODO: cleanup cache when all documents are disposed
 		}
+		await familyCache.initializedPromise;
 
 		return familyCache;
 	};
@@ -192,10 +227,10 @@ export class EntityStore {
 
 		const snapshot = entity?.getSnapshot();
 		if (snapshot) {
-			const stored = cloneDeep(snapshot);
-			assignIndexValues(this.schema.collections[collection], stored);
-			// IMPORTANT! this property must be assigned
-			assignOidPropertiesToAllSubObjects(stored);
+			const stored = getIndexValues(
+				this.schema.collections[collection],
+				snapshot,
+			);
 			try {
 				const tx = this.db.transaction(collection, 'readwrite');
 				const store = tx.objectStore(collection);
@@ -221,11 +256,11 @@ export class EntityStore {
 
 	get = async (oid: ObjectIdentifier) => {
 		const familyCache = await this.openFamilyCache(oid);
-		const schema = this.getDocumentSchema(oid);
+		const { schema, readonlyKeys } = this.getDocumentSchema(oid);
 		if (!schema) {
 			return null;
 		}
-		return familyCache.getEntity(oid, schema);
+		return familyCache.getEntity(oid, schema, undefined, readonlyKeys);
 	};
 
 	/**
@@ -236,11 +271,11 @@ export class EntityStore {
 	getCached = (oid: ObjectIdentifier) => {
 		const cache = this.documentFamilyCaches.get(oid);
 		if (cache) {
-			const schema = this.getDocumentSchema(oid);
+			const { schema, readonlyKeys } = this.getDocumentSchema(oid);
 			if (!schema) {
 				return null;
 			}
-			return cache.getEntity(oid, schema);
+			return cache.getEntity(oid, schema, undefined, readonlyKeys);
 		}
 		return null;
 	};
@@ -268,7 +303,7 @@ export class EntityStore {
 		// only holding it in memory would introduce lag before it shows up
 		// in other queries.
 		await this.submitOperations(operations, options);
-		const schema = this.getDocumentSchema(oid);
+		const { schema, readonlyKeys } = this.getDocumentSchema(oid);
 		if (!schema) {
 			throw new Error(
 				`Cannot create a document in the ${
@@ -276,7 +311,7 @@ export class EntityStore {
 				} collection; it is not defined in the current schema version.`,
 			);
 		}
-		return familyCache.getEntity(oid, schema);
+		return familyCache.getEntity(oid, schema, undefined, readonlyKeys);
 	};
 
 	private addOperationsToOpenCaches = async (
@@ -384,6 +419,10 @@ export class EntityStore {
 		baselines: DocumentBaseline[];
 		reset?: boolean;
 	}) => {
+		if (this._disposed) {
+			this.log('warn', 'EntityStore is disposed, not adding data');
+			return;
+		}
 		// convert operations to tagged operations with confirmed = false
 		// while we process and store them. this is in-place so as to
 		// not allocate a bunch of objects...
@@ -427,7 +466,7 @@ export class EntityStore {
 		await this.meta.insertRemoteOperations(operations);
 
 		if (reset) {
-			await this.refreshAllCaches(true);
+			await this.refreshAllCaches(true, true);
 		}
 
 		// recompute all affected documents for querying
@@ -511,6 +550,10 @@ export class EntityStore {
 		await this.operationBatcher.flush(this.currentBatchKey);
 	};
 
+	flushAllBatches = async () => {
+		await Promise.all(this.operationBatcher.flushAll());
+	};
+
 	private flushOperations = async (
 		operations: Operation[],
 		batchKey: string,
@@ -632,7 +675,7 @@ export class EntityStore {
 		// );
 	};
 
-	destroy = () => {
+	destroy = async () => {
 		this._disposed = true;
 		for (const unsubscribe of this.unsubscribes) {
 			unsubscribe();
@@ -641,6 +684,7 @@ export class EntityStore {
 			cache.dispose();
 		}
 		this.documentFamilyCaches.clear();
+		await this.flushAllBatches();
 	};
 
 	private handleRebase = (baselines: DocumentBaseline[]) => {
@@ -661,9 +705,12 @@ export class EntityStore {
 		}
 	};
 
-	private refreshAllCaches = async (dropUnconfirmed = false) => {
+	private refreshAllCaches = async (
+		dropUnconfirmed = false,
+		dropAll = false,
+	) => {
 		for (const [_, cache] of this.documentFamilyCaches) {
-			await this.refreshFamilyCache(cache, dropUnconfirmed);
+			await this.refreshFamilyCache(cache, dropUnconfirmed, dropAll);
 		}
 	};
 }

package/src/idb.ts

@@ -105,3 +105,13 @@ export async function deleteAllDatabases(
 	]);
 	window.location.reload();
 }
+
+export function deleteDatabase(name: string, indexedDB = window.indexedDB) {
+	return storeRequestPromise(indexedDB.deleteDatabase(name));
+}
+
+export async function getAllDatabaseNamesAndVersions(
+	indexedDB: IDBFactory = window.indexedDB,
+) {
+	return indexedDB.databases();
+}

package/src/index.ts

@@ -28,6 +28,7 @@ export {
 	schema,
 	createDefaultMigration,
 	migrate,
+	createMigration,
 } from '@verdant-web/common';
 export type {
 	StorageDocument,

package/src/metadata/BaselinesStore.ts

@@ -1,6 +1,7 @@
 import {
 	DocumentBaseline,
-	getOidRange,
+	getOidSubIdRange,
+	getOidRoot,
 	ObjectIdentifier,
 } from '@verdant-web/common';
 import { IDBService } from '../IDBService.js';
@@ -38,8 +39,14 @@ export class BaselinesStore extends IDBService {
 		return this.iterate(
 			'baselines',
 			(store) => {
-				const [start, end] = getOidRange(oid);
-				return store.openCursor(IDBKeyRange.bound(start, end, false, false));
+				const root = getOidRoot(oid);
+				const [start, end] = getOidSubIdRange(oid);
+				return [
+					// first the root itself
+					store.openCursor(IDBKeyRange.only(root)),
+					// then the range of its possible subdocuments
+					store.openCursor(IDBKeyRange.bound(start, end, false, false)),
+				];
 			},
 			iterator,
 			mode,
@@ -54,9 +61,13 @@ export class BaselinesStore extends IDBService {
 		const result = await this.runAll<DocumentBaseline[]>(
 			'baselines',
 			(store) => {
-				return docOids.map((oid) => {
-					const [start, end] = getOidRange(oid);
-					return store.getAll(IDBKeyRange.bound(start, end, false, false));
+				return docOids.flatMap((oid) => {
+					const root = getOidRoot(oid);
+					const [start, end] = getOidSubIdRange(oid);
+					return [
+						store.get(root),
+						store.getAll(IDBKeyRange.bound(start, end, false, false)),
+					];
 				});
 			},
 			mode,

package/src/metadata/openMetadataDatabase.ts

@@ -1,20 +1,19 @@
+import { closeDatabase, storeRequestPromise } from '../idb.js';
+
 const migrations = [version1, version2, version3, version4];
 
-export function openMetadataDatabase(
-	namespace: string,
-	{
-		indexedDB = window.indexedDB,
-		databaseName,
-		log,
-	}: {
-		indexedDB?: IDBFactory;
-		databaseName: string;
-		log?: (...args: any[]) => void;
-	},
-): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
+export function openMetadataDatabase({
+	indexedDB = window.indexedDB,
+	namespace,
+	log,
+}: {
+	indexedDB?: IDBFactory;
+	namespace: string;
+	log?: (...args: any[]) => void;
+}): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
 	return new Promise<{ wasInitialized: boolean; db: IDBDatabase }>(
 		(resolve, reject) => {
-			const request = indexedDB.open(databaseName, 4);
+			const request = indexedDB.open(`${namespace}_meta`, 4);
 			let wasInitialized = false;
 			request.onupgradeneeded = async (event) => {
 				const db = request.result;
@@ -40,6 +39,90 @@ export function openMetadataDatabase(
 	);
 }
 
+export async function openWIPMetadataDatabase({
+	wipNamespace,
+	namespace,
+	indexedDB,
+	log,
+}: {
+	indexedDB?: IDBFactory;
+	namespace: string;
+	wipNamespace: string;
+	log?: (...args: any[]) => void;
+}): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
+	const result = await openMetadataDatabase({
+		namespace: wipNamespace,
+		indexedDB,
+		log,
+	});
+
+	// this WIP database was already set up.
+	if (!result.wasInitialized) {
+		return result;
+	}
+
+	log?.('debug', 'Beginning copy of production metadata database to WIP');
+	// copy all data from production metadata database
+	const { db: prodDb } = await openMetadataDatabase({
+		namespace,
+		indexedDB,
+		log,
+	});
+
+	const tx = prodDb.transaction(
+		['baselines', 'operations', 'info'],
+		'readonly',
+	);
+	const [baselines, operations, info] = await Promise.all([
+		storeRequestPromise(tx.objectStore('baselines').getAll()),
+		storeRequestPromise(tx.objectStore('operations').getAll()),
+		storeRequestPromise(tx.objectStore('info').getAll()),
+	]);
+
+	const wipTx = result.db.transaction(
+		['baselines', 'operations', 'info'],
+		'readwrite',
+	);
+	const wipBaselines = wipTx.objectStore('baselines');
+	const wipOperations = wipTx.objectStore('operations');
+	const wipInfo = wipTx.objectStore('info');
+
+	for (const baseline of baselines) {
+		wipBaselines.put(baseline);
+	}
+	for (const operation of operations) {
+		wipOperations.put(operation);
+	}
+	for (const infoItem of info) {
+		wipInfo.put(infoItem);
+	}
+
+	await new Promise<void>((resolve, reject) => {
+		wipTx.oncomplete = () => {
+			resolve();
+		};
+		wipTx.onerror = (event) => {
+			reject(event);
+		};
+		wipTx.onabort = (event) => {
+			reject(event);
+		};
+	});
+
+	await closeDatabase(prodDb);
+
+	log?.(
+		'debug',
+		'Finished copy of production metadata database to WIP. Copied:',
+		baselines.length,
+		'baselines,',
+		operations.length,
+		'operations',
+	);
+
+	return result;
+}
+
 async function version1(db: IDBDatabase, tx: IDBTransaction) {
 	const baselinesStore = db.createObjectStore('baselines', {
 		keyPath: 'oid',

package/src/migration/db.ts

@@ -114,7 +114,9 @@ export async function openDatabase(
 	indexedDb: IDBFactory,
 	namespace: string,
 	version: number,
+	log?: (...args: any[]) => void,
 ): Promise<IDBDatabase> {
+	log?.('debug', 'Opening database', namespace, 'at version', version);
 	const db = await new Promise<IDBDatabase>((resolve, reject) => {
 		const request = indexedDb.open(
 			[namespace, 'collections'].join('_'),
@@ -124,8 +126,19 @@ export async function openDatabase(
 			const transaction = request.transaction!;
 			transaction.abort();
 
+			log?.(
+				'error',
+				'Database upgrade needed, but not expected',
+				'Expected',
+				version,
+				'Got',
+				request.result.version,
+			);
 			reject(
-				new Error('Migration error: database version changed while migrating'),
+				request.error ||
+					new Error(
+						`Migration error: database version changed unexpectedly when reading current data. Expected ${version}, got ${request.result.version}`,
+					),
 			);
 		};
 		request.onsuccess = (event) => {