@verdant-web/store 2.5.7 → 2.6.0

package/src/entities/DocumentFamiliyCache.ts

@@ -14,6 +14,7 @@ import type { EntityStore } from './EntityStore.js';
 import { WeakRef } from './FakeWeakRef.js';
 import { Context } from '../context.js';
 import { TaggedOperation } from '../types.js';
+import { Resolvable } from '../utils/Resolvable.js';
 
 /**
  * Local operations: operations on this client that haven't
@@ -44,6 +45,14 @@ export class DocumentFamilyCache extends EventSubscriber<
 	private context;
 	private storeTools: StoreTools;
 
+	private _initialized = new Resolvable<boolean>();
+	get initializedPromise() {
+		return this._initialized.promise;
+	}
+	setInitialized = () => {
+		this._initialized.resolve(true);
+	};
+
 	constructor({
 		oid,
 		store,
@@ -236,6 +245,14 @@ export class DocumentFamilyCache extends EventSubscriber<
 	};
 
 	computeView = (oid: ObjectIdentifier) => {
+		if (
+			this.baselinesMap.size === 0 &&
+			this.operationsMap.size === 0 &&
+			this.localOperationsMap.size === 0
+		) {
+			this.context.log('debug', `Entity ${oid} accessed with no data at all`);
+			return { view: null, deleted: true, lastTimestamp: null };
+		}
 		const confirmed = this.computeConfirmedView(oid);
 		const unconfirmedOperations = this.localOperationsMap.get(oid) || [];
 		if (confirmed.empty && !unconfirmedOperations.length) {
@@ -331,33 +348,56 @@ export class DocumentFamilyCache extends EventSubscriber<
 		this.entities.clear();
 	};
 
-	reset = (
-		operations: TaggedOperation[],
-		baselines: DocumentBaseline[],
+	reset = ({
+		operations,
+		baselines,
+		dropExistingUnconfirmed: dropUnconfirmed = false,
+		unconfirmedOperations,
+		dropAll,
+	}: {
+		operations: TaggedOperation[];
+		unconfirmedOperations?: Operation[];
+		baselines: DocumentBaseline[];
 		/**
 		 * Whether to drop operations which are only in-memory. Unconfirmed operations
 		 * will not be restored from storage until they are persisted, so it's not advisable
 		 * to use this unless the intention is to completely clear the entities.
 		 */
-		dropUnconfirmed = false,
-	) => {
-		const info = { isLocal: false, affectedOids: new Set<ObjectIdentifier>() };
-		this.baselinesMap = new Map(
-			baselines.map((baseline) => [baseline.oid, baseline]),
+		dropExistingUnconfirmed?: boolean;
+		/**
+		 * Drop unconfirmed and confirmed data before resetting to incoming data.
+		 * This is dangerous due to race conditions. Only use when a full reset is
+		 * required.
+		 */
+		dropAll?: boolean;
+	}) => {
+		this.context.log(
+			'debug',
+			`Resetting cache for ${this.oid} with ${operations.length} ops and ${baselines.length} baselines, dropUnconfirmed=${dropUnconfirmed}`,
 		);
-		if (dropUnconfirmed) {
-			this.operationsMap = new Map();
-		} else {
-			// clear out all confirmed operations, leaving only unconfirmed
-			// which have been added in memory but not yet persisted in storage
-			for (const oid of this.operationsMap.keys()) {
-				this.operationsMap.set(
-					oid,
-					this.operationsMap.get(oid)?.filter((op) => !op.confirmed) ?? [],
-				);
+		const info = { isLocal: false, affectedOids: new Set<ObjectIdentifier>() };
+
+		// NOTE: not clearing these maps... there are even more
+		// race conditions where we begin opening a cache, queue up a
+		// reset, then receive incoming operations before the reset
+		// actually hits this function, so those incoming ops are
+		// dropped.
+		// FIXME: include this in a future refactor of this
+		// whole system.
+
+		if (dropAll) this.baselinesMap.clear();
+		this.insertBaselines(baselines, info);
+
+		if (dropAll) this.operationsMap.clear();
+		this.insertOperations(operations, info);
+
+		if (unconfirmedOperations || dropUnconfirmed) {
+			this.localOperationsMap.clear();
+			if (unconfirmedOperations) {
+				this.insertLocalOperations(unconfirmedOperations);
 			}
 		}
-		this.insertOperations(operations, info);
+
 		for (const oid of this.entities.keys()) {
 			const entityRef = this.entities.get(oid);
 			const entity = entityRef?.deref();

package/src/entities/Entity.ts

@@ -347,6 +347,23 @@ export class Entity<
 	};
 
 	protected processInputValue = (value: any, key: any) => {
+		// disassociate incoming OIDs on values and generally break object
+		// references. cloning doesn't work on files so those are
+		// filtered out.
+		// The goal here is to be safe about a bunch of cases that could
+		// result in corrupt data, like...
+		// ent1.set('objField', ent2.get('objField'))
+		// or
+		// var shared = { foo: 'bar' };
+		// ent1.set('objField', shared);
+		// ent2.set('objField', shared);
+		// ... each of these would result in the same object being
+		// referenced in multiple entities, which could mean introduction
+		// of foreign OIDs, or one object being assigned different OIDs
+		// with unexpected results.
+		if (!(value instanceof File)) {
+			value = cloneDeep(value, false);
+		}
 		const fieldSchema = this.getChildFieldSchema(key);
 		if (fieldSchema) {
 			traverseCollectionFieldsAndApplyDefaults(value, fieldSchema);

package/src/entities/EntityStore.ts

@@ -113,9 +113,16 @@ export class EntityStore {
 	private refreshFamilyCache = async (
 		familyCache: DocumentFamilyCache,
 		dropUnconfirmed = false,
+		dropAll = false,
 	) => {
 		// avoid writing to disposed db
-		if (this._disposed) return;
+		if (this._disposed) {
+			this.context.log(
+				'debug',
+				`EntityStore is disposed, not refreshing ${familyCache.oid} cache`,
+			);
+			return;
+		}
 
 		// metadata must be loaded from database to initialize family cache
 		const transaction = this.meta.createTransaction([
@@ -146,21 +153,40 @@ export class EntityStore {
 				{ transaction, mode: 'readwrite' },
 			),
 		]);
-		familyCache.reset(operations, baselines, dropUnconfirmed);
+		familyCache.reset({
+			operations,
+			baselines,
+			dropExistingUnconfirmed: dropUnconfirmed,
+			dropAll,
+		});
 	};
 
 	private openFamilyCache = async (oid: ObjectIdentifier) => {
 		const documentOid = getOidRoot(oid);
 		let familyCache = this.documentFamilyCaches.get(documentOid);
 		if (!familyCache) {
+			this.context.log('debug', 'opening family cache for', documentOid);
 			// metadata must be loaded from database to initialize family cache
 			familyCache = new DocumentFamilyCache({
 				oid: documentOid,
 				store: this,
 				context: this.context,
 			});
+
+			// PROBLEM: because the next line is async, it yields to
+			// queued promises which may need data from this cache,
+			// but the cache is empty. But if we move the set to
+			// after the async, we can clobber an existing cache
+			// with race conditions...
+			// So as an attempt to fix that, I've added a promise
+			// on DocumentFamilyCache which I manually resolve
+			// with setInitialized, then await initializedPromise
+			// further down even if there was a cache hit.
+			// Surely there is a better pattern for this.
+			// FIXME:
 			this.documentFamilyCaches.set(documentOid, familyCache);
 			await this.refreshFamilyCache(familyCache);
+			familyCache.setInitialized();
 
 			// this.unsubscribes.push(
 			// 	familyCache.subscribe('change:*', this.onEntityChange),
@@ -168,6 +194,7 @@ export class EntityStore {
 
 			// TODO: cleanup cache when all documents are disposed
 		}
+		await familyCache.initializedPromise;
 
 		return familyCache;
 	};
@@ -384,6 +411,10 @@ export class EntityStore {
 		baselines: DocumentBaseline[];
 		reset?: boolean;
 	}) => {
+		if (this._disposed) {
+			this.log('warn', 'EntityStore is disposed, not adding data');
+			return;
+		}
 		// convert operations to tagged operations with confirmed = false
 		// while we process and store them. this is in-place so as to
 		// not allocate a bunch of objects...
@@ -427,7 +458,7 @@ export class EntityStore {
 		await this.meta.insertRemoteOperations(operations);
 
 		if (reset) {
-			await this.refreshAllCaches(true);
+			await this.refreshAllCaches(true, true);
 		}
 
 		// recompute all affected documents for querying
@@ -511,6 +542,10 @@ export class EntityStore {
 		await this.operationBatcher.flush(this.currentBatchKey);
 	};
 
+	flushAllBatches = async () => {
+		await Promise.all(this.operationBatcher.flushAll());
+	};
+
 	private flushOperations = async (
 		operations: Operation[],
 		batchKey: string,
@@ -632,7 +667,7 @@ export class EntityStore {
 		// );
 	};
 
-	destroy = () => {
+	destroy = async () => {
 		this._disposed = true;
 		for (const unsubscribe of this.unsubscribes) {
 			unsubscribe();
@@ -641,6 +676,7 @@ export class EntityStore {
 			cache.dispose();
 		}
 		this.documentFamilyCaches.clear();
+		await this.flushAllBatches();
 	};
 
 	private handleRebase = (baselines: DocumentBaseline[]) => {
@@ -661,9 +697,12 @@ export class EntityStore {
 		}
 	};
 
-	private refreshAllCaches = async (dropUnconfirmed = false) => {
+	private refreshAllCaches = async (
+		dropUnconfirmed = false,
+		dropAll = false,
+	) => {
 		for (const [_, cache] of this.documentFamilyCaches) {
-			await this.refreshFamilyCache(cache, dropUnconfirmed);
+			await this.refreshFamilyCache(cache, dropUnconfirmed, dropAll);
 		}
 	};
 }

package/src/idb.ts

@@ -105,3 +105,13 @@ export async function deleteAllDatabases(
 	]);
 	window.location.reload();
 }
+
+export function deleteDatabase(name: string, indexedDB = window.indexedDB) {
+	return storeRequestPromise(indexedDB.deleteDatabase(name));
+}
+
+export async function getAllDatabaseNamesAndVersions(
+	indexedDB: IDBFactory = window.indexedDB,
+) {
+	return indexedDB.databases();
+}

package/src/metadata/Metadata.ts

@@ -141,7 +141,10 @@ export class Metadata extends EventSubscriber<{
 		return Array.from(oids);
 	};
 
-	getDocumentSnapshot = async (oid: ObjectIdentifier) => {
+	getDocumentSnapshot = async (
+		oid: ObjectIdentifier,
+		options: { to?: string } = {},
+	) => {
 		const documentOid = getOidRoot(oid);
 		assert(documentOid === oid, 'Must be root document OID');
 		const transaction = this.db.transaction(
@@ -170,6 +173,8 @@ export class Metadata extends EventSubscriber<{
 			},
 			{
 				transaction,
+				// only apply operations up to the current time
+				to: options.to || this.now,
 			},
 		);
 		const root = objectMap.get(documentOid);

package/src/metadata/openMetadataDatabase.ts

@@ -1,20 +1,19 @@
+import { closeDatabase, storeRequestPromise } from '../idb.js';
+
 const migrations = [version1, version2, version3, version4];
 
-export function openMetadataDatabase(
-	namespace: string,
-	{
-		indexedDB = window.indexedDB,
-		databaseName,
-		log,
-	}: {
-		indexedDB?: IDBFactory;
-		databaseName: string;
-		log?: (...args: any[]) => void;
-	},
-): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
+export function openMetadataDatabase({
+	indexedDB = window.indexedDB,
+	namespace,
+	log,
+}: {
+	indexedDB?: IDBFactory;
+	namespace: string;
+	log?: (...args: any[]) => void;
+}): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
 	return new Promise<{ wasInitialized: boolean; db: IDBDatabase }>(
 		(resolve, reject) => {
-			const request = indexedDB.open(databaseName, 4);
+			const request = indexedDB.open(`${namespace}_meta`, 4);
 			let wasInitialized = false;
 			request.onupgradeneeded = async (event) => {
 				const db = request.result;
@@ -40,6 +39,90 @@ export function openMetadataDatabase(
 	);
 }
 
+export async function openWIPMetadataDatabase({
+	wipNamespace,
+	namespace,
+	indexedDB,
+	log,
+}: {
+	indexedDB?: IDBFactory;
+	namespace: string;
+	wipNamespace: string;
+	log?: (...args: any[]) => void;
+}): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
+	const result = await openMetadataDatabase({
+		namespace: wipNamespace,
+		indexedDB,
+		log,
+	});
+
+	// this WIP database was already set up.
+	if (!result.wasInitialized) {
+		return result;
+	}
+
+	log?.('debug', 'Beginning copy of production metadata database to WIP');
+	// copy all data from production metadata database
+	const { db: prodDb } = await openMetadataDatabase({
+		namespace,
+		indexedDB,
+		log,
+	});
+
+	const tx = prodDb.transaction(
+		['baselines', 'operations', 'info'],
+		'readonly',
+	);
+	const [baselines, operations, info] = await Promise.all([
+		storeRequestPromise(tx.objectStore('baselines').getAll()),
+		storeRequestPromise(tx.objectStore('operations').getAll()),
+		storeRequestPromise(tx.objectStore('info').getAll()),
+	]);
+
+	const wipTx = result.db.transaction(
+		['baselines', 'operations', 'info'],
+		'readwrite',
+	);
+	const wipBaselines = wipTx.objectStore('baselines');
+	const wipOperations = wipTx.objectStore('operations');
+	const wipInfo = wipTx.objectStore('info');
+
+	for (const baseline of baselines) {
+		wipBaselines.put(baseline);
+	}
+	for (const operation of operations) {
+		wipOperations.put(operation);
+	}
+	for (const infoItem of info) {
+		wipInfo.put(infoItem);
+	}
+
+	await new Promise<void>((resolve, reject) => {
+		wipTx.oncomplete = () => {
+			resolve();
+		};
+		wipTx.onerror = (event) => {
+			reject(event);
+		};
+		wipTx.onabort = (event) => {
+			reject(event);
+		};
+	});
+
+	await closeDatabase(prodDb);
+
+	log?.(
+		'debug',
+		'Finished copy of production metadata database to WIP. Copied:',
+		baselines.length,
+		'baselines,',
+		operations.length,
+		'operations',
+	);
+
+	return result;
+}
+
 async function version1(db: IDBDatabase, tx: IDBTransaction) {
 	const baselinesStore = db.createObjectStore('baselines', {
 		keyPath: 'oid',

package/src/migration/db.ts

@@ -114,7 +114,9 @@ export async function openDatabase(
 	indexedDb: IDBFactory,
 	namespace: string,
 	version: number,
+	log?: (...args: any[]) => void,
 ): Promise<IDBDatabase> {
+	log?.('debug', 'Opening database', namespace, 'at version', version);
 	const db = await new Promise<IDBDatabase>((resolve, reject) => {
 		const request = indexedDb.open(
 			[namespace, 'collections'].join('_'),
@@ -124,8 +126,19 @@ export async function openDatabase(
 			const transaction = request.transaction!;
 			transaction.abort();
 
+			log?.(
+				'error',
+				'Database upgrade needed, but not expected',
+				'Expected',
+				version,
+				'Got',
+				request.result.version,
+			);
 			reject(
-				new Error('Migration error: database version changed while migrating'),
+				request.error ||
+					new Error(
+						`Migration error: database version changed unexpectedly when reading current data. Expected ${version}, got ${request.result.version}`,
+					),
 			);
 		};
 		request.onsuccess = (event) => {