@verdant-web/store 2.5.8 → 2.7.0

package/src/client/ClientDescriptor.ts

@@ -1,13 +1,29 @@
-import { EventSubscriber, Migration, StorageSchema } from '@verdant-web/common';
+import {
+	EventSubscriber,
+	Migration,
+	StorageSchema,
+	hashObject,
+} from '@verdant-web/common';
 import { Context } from '../context.js';
 import { FileManagerConfig } from '../files/FileManager.js';
 import { Metadata } from '../metadata/Metadata.js';
-import { openMetadataDatabase } from '../metadata/openMetadataDatabase.js';
-import { openDocumentDatabase } from '../migration/openDatabase.js';
+import {
+	openMetadataDatabase,
+	openWIPMetadataDatabase,
+} from '../metadata/openMetadataDatabase.js';
+import {
+	openDocumentDatabase,
+	openWIPDocumentDatabase,
+} from '../migration/openDatabase.js';
 import { ServerSyncOptions } from '../sync/Sync.js';
 import { UndoHistory } from '../UndoHistory.js';
 import { Client } from './Client.js';
-import { deleteAllDatabases } from '../idb.js';
+import {
+	deleteAllDatabases,
+	deleteDatabase,
+	getAllDatabaseNamesAndVersions,
+} from '../idb.js';
+import { FakeWeakRef } from '../FakeWeakRef.js';
 
 export interface ClientDescriptorOptions<Presence = any, Profile = any> {
 	/** The schema used to create this client */
@@ -43,6 +59,14 @@ export interface ClientDescriptorOptions<Presence = any, Profile = any> {
 	 * Configuration for file management
 	 */
 	files?: FileManagerConfig;
+	/**
+	 * Enables experimental WeakRef usage to cull documents
+	 * from cache that aren't being used. This is a performance
+	 * optimization which has been tested under all Verdant's test
+	 * suites but I still want to keep testing it in the real world
+	 * before turning it on.
+	 */
+	EXPERIMENTAL_weakRefs?: boolean;
 }
 
 /**
@@ -50,13 +74,17 @@ export interface ClientDescriptorOptions<Presence = any, Profile = any> {
  * Storage creation promise and exposes some metadata which can
  * be useful immediately.
  */
-export class ClientDescriptor<Presence = any, Profile = any> {
-	private readonly _readyPromise: Promise<Client>;
+export class ClientDescriptor<
+	Presence = any,
+	Profile = any,
+	ClientImpl extends Client = Client,
+> {
+	private readonly _readyPromise: Promise<ClientImpl>;
 	// assertions because these are defined by plucking them from
 	// Promise initializer
-	private resolveReady!: (storage: Client) => void;
+	private resolveReady!: (storage: ClientImpl) => void;
 	private rejectReady!: (err: Error) => void;
-	private _resolvedValue: Client | undefined;
+	private _resolvedValue: ClientImpl | undefined;
 	private _initializing = false;
 	private _namespace: string;
 
@@ -88,51 +116,13 @@ export class ClientDescriptor<Presence = any, Profile = any> {
 		}
 		this._initializing = true;
 		try {
-			const metaDbName = [init.namespace, 'meta'].join('_');
-			const { db: metaDb } = await openMetadataDatabase(this._namespace, {
-				indexedDB: init.indexedDb,
-				log: init.log,
-				databaseName: metaDbName,
-			});
-
-			const context: Omit<Context, 'documentDb'> = {
-				namespace: this._namespace,
-				metaDb,
-				schema: init.schema,
-				log: init.log || (() => {}),
-				undoHistory: init.undoHistory || new UndoHistory(),
-				entityEvents: new EventSubscriber(),
-				globalEvents: new EventSubscriber(),
-			};
-			const meta = new Metadata({
-				context,
-				disableRebasing: init.disableRebasing,
-			});
-
-			// verify schema integrity
-			await meta.updateSchema(init.schema, init.overrideSchemaConflict);
-
-			const documentDb = await openDocumentDatabase({
-				context,
-				version: init.schema.version,
-				meta,
-				migrations: init.migrations,
-				indexedDB: init.indexedDb,
-			});
-
-			const fullContext: Context = Object.assign(context, { documentDb });
-
-			const storage = new Client(
-				{
-					syncConfig: init.sync,
-					migrations: init.migrations,
-					files: init.files,
-				},
-				fullContext,
-				{
-					meta,
-				},
-			);
+			let storage: ClientImpl;
+			if (init.schema.wip) {
+				storage = await this.initializeWIPDatabases(init);
+			} else {
+				storage = await this.initializeDatabases(init);
+				this.cleanupWIPDatabases(init);
+			}
 
 			this.resolveReady(storage);
 			this._resolvedValue = storage;
@@ -145,6 +135,139 @@ export class ClientDescriptor<Presence = any, Profile = any> {
 		}
 	};
 
+	private initializeDatabases = async (init: ClientDescriptorOptions) => {
+		const { db: metaDb } = await openMetadataDatabase({
+			indexedDB: init.indexedDb,
+			log: init.log,
+			namespace: init.namespace,
+		});
+
+		const context: Omit<Context, 'documentDb'> = {
+			namespace: this._namespace,
+			metaDb,
+			schema: init.schema,
+			log: init.log || (() => {}),
+			undoHistory: init.undoHistory || new UndoHistory(),
+			entityEvents: new EventSubscriber(),
+			globalEvents: new EventSubscriber(),
+			weakRef: (value) => {
+				if (init.EXPERIMENTAL_weakRefs) {
+					return new WeakRef(value);
+				} else {
+					return new FakeWeakRef(value) as unknown as WeakRef<typeof value>;
+				}
+			},
+		};
+		const meta = new Metadata({
+			context,
+			disableRebasing: init.disableRebasing,
+		});
+
+		// verify schema integrity
+		await meta.updateSchema(init.schema, init.overrideSchemaConflict);
+
+		const documentDb = await openDocumentDatabase({
+			context,
+			version: init.schema.version,
+			meta,
+			migrations: init.migrations,
+			indexedDB: init.indexedDb,
+		});
+
+		const fullContext: Context = Object.assign(context, { documentDb });
+
+		const storage = new Client(
+			{
+				syncConfig: init.sync,
+				migrations: init.migrations,
+				files: init.files,
+			},
+			fullContext,
+			{
+				meta,
+			},
+		) as ClientImpl;
+
+		return storage;
+	};
+
+	private initializeWIPDatabases = async (init: ClientDescriptorOptions) => {
+		const schemaHash = hashObject(init.schema);
+		console.info(`WIP schema in use. Opening database with hash ${schemaHash}`);
+
+		const wipNamespace = `@@wip_${init.namespace}_${schemaHash}`;
+		const { db: metaDb } = await openWIPMetadataDatabase({
+			indexedDB: init.indexedDb,
+			log: init.log,
+			namespace: init.namespace,
+			wipNamespace: wipNamespace,
+		});
+
+		const context: Omit<Context, 'documentDb'> = {
+			namespace: this._namespace,
+			metaDb,
+			schema: init.schema,
+			log: init.log || (() => {}),
+			undoHistory: init.undoHistory || new UndoHistory(),
+			entityEvents: new EventSubscriber(),
+			globalEvents: new EventSubscriber(),
+			weakRef: (value) => {
+				if (init.EXPERIMENTAL_weakRefs) {
+					return new WeakRef(value);
+				} else {
+					return new FakeWeakRef(value) as unknown as WeakRef<typeof value>;
+				}
+			},
+		};
+		const meta = new Metadata({
+			context,
+			disableRebasing: init.disableRebasing,
+		});
+
+		// verify schema integrity
+		await meta.updateSchema(init.schema, init.overrideSchemaConflict);
+
+		const documentDb = await openWIPDocumentDatabase({
+			context,
+			version: init.schema.version,
+			meta,
+			migrations: init.migrations,
+			indexedDB: init.indexedDb,
+			wipNamespace,
+		});
+
+		const fullContext: Context = Object.assign(context, { documentDb });
+
+		const storage = new Client(
+			{
+				syncConfig: init.sync,
+				migrations: init.migrations,
+				files: init.files,
+			},
+			fullContext,
+			{
+				meta,
+			},
+		) as ClientImpl;
+
+		return storage;
+	};
+
+	private cleanupWIPDatabases = async (init: ClientDescriptorOptions) => {
+		const databaseInfo = await getAllDatabaseNamesAndVersions(init.indexedDb);
+		const wipDatabases = databaseInfo
+			.filter((db) => db.name?.startsWith('@@wip_'))
+			.map((db) => db.name!);
+		// don't clear a current WIP database.
+		const wipDatabasesToDelete = wipDatabases.filter(
+			(db) =>
+				!db.startsWith(`@@wip_${init.namespace}_${hashObject(init.schema)}`),
+		);
+		for (const db of wipDatabasesToDelete) {
+			await deleteDatabase(db, init.indexedDb);
+		}
+	};
+
 	get current() {
 		// exposing an immediate value if already resolved lets us
 		// skip the promise microtask when accessing this externally if

package/src/context.ts

@@ -26,4 +26,5 @@ export interface Context {
 		 */
 		futureSeen: (timestamp: string) => void;
 	}>;
+	weakRef<T extends object>(value: T): WeakRef<T>;
 }

package/src/entities/DocumentFamiliyCache.ts

@@ -11,9 +11,9 @@ import {
 } from '@verdant-web/common';
 import { Entity, refreshEntity, StoreTools } from './Entity.js';
 import type { EntityStore } from './EntityStore.js';
-import { WeakRef } from './FakeWeakRef.js';
 import { Context } from '../context.js';
 import { TaggedOperation } from '../types.js';
+import { Resolvable } from '../utils/Resolvable.js';
 
 /**
  * Local operations: operations on this client that haven't
@@ -44,6 +44,14 @@ export class DocumentFamilyCache extends EventSubscriber<
 	private context;
 	private storeTools: StoreTools;
 
+	private _initialized = new Resolvable<boolean>();
+	get initializedPromise() {
+		return this._initialized.promise;
+	}
+	setInitialized = () => {
+		this._initialized.resolve(true);
+	};
+
 	constructor({
 		oid,
 		store,
@@ -69,6 +77,10 @@ export class DocumentFamilyCache extends EventSubscriber<
 		this.context = context;
 	}
 
+	get weakRef() {
+		return this.context.weakRef;
+	}
+
 	insertLocalOperations = (operations: Operation[]) => {
 		const oidSet = new Set<ObjectIdentifier>();
 		for (const operation of operations) {
@@ -236,6 +248,14 @@ export class DocumentFamilyCache extends EventSubscriber<
 	};
 
 	computeView = (oid: ObjectIdentifier) => {
+		if (
+			this.baselinesMap.size === 0 &&
+			this.operationsMap.size === 0 &&
+			this.localOperationsMap.size === 0
+		) {
+			this.context.log('debug', `Entity ${oid} accessed with no data at all`);
+			return { view: null, deleted: true, lastTimestamp: null };
+		}
 		const confirmed = this.computeConfirmedView(oid);
 		const unconfirmedOperations = this.localOperationsMap.get(oid) || [];
 		if (confirmed.empty && !unconfirmedOperations.length) {
@@ -303,6 +323,7 @@ export class DocumentFamilyCache extends EventSubscriber<
 		oid: ObjectIdentifier,
 		schema: StorageFieldSchema,
 		parent?: Entity,
+		readonlyKeys?: string[],
 	): Entity => {
 		let entityRef = this.entities.get(oid);
 		let entity = entityRef?.deref();
@@ -313,10 +334,11 @@ export class DocumentFamilyCache extends EventSubscriber<
 				fieldSchema: schema,
 				store: this.storeTools,
 				parent,
+				readonlyKeys,
 			});
 
 			// immediately add to cache and queue a removal if nobody subscribed
-			this.entities.set(oid, new WeakRef(entity));
+			this.entities.set(oid, this.context.weakRef(entity));
 		}
 
 		return entity as any;
@@ -331,33 +353,56 @@ export class DocumentFamilyCache extends EventSubscriber<
 		this.entities.clear();
 	};
 
-	reset = (
-		operations: TaggedOperation[],
-		baselines: DocumentBaseline[],
+	reset = ({
+		operations,
+		baselines,
+		dropExistingUnconfirmed: dropUnconfirmed = false,
+		unconfirmedOperations,
+		dropAll,
+	}: {
+		operations: TaggedOperation[];
+		unconfirmedOperations?: Operation[];
+		baselines: DocumentBaseline[];
 		/**
 		 * Whether to drop operations which are only in-memory. Unconfirmed operations
 		 * will not be restored from storage until they are persisted, so it's not advisable
 		 * to use this unless the intention is to completely clear the entities.
 		 */
-		dropUnconfirmed = false,
-	) => {
-		const info = { isLocal: false, affectedOids: new Set<ObjectIdentifier>() };
-		this.baselinesMap = new Map(
-			baselines.map((baseline) => [baseline.oid, baseline]),
+		dropExistingUnconfirmed?: boolean;
+		/**
+		 * Drop unconfirmed and confirmed data before resetting to incoming data.
+		 * This is dangerous due to race conditions. Only use when a full reset is
+		 * required.
+		 */
+		dropAll?: boolean;
+	}) => {
+		this.context.log(
+			'debug',
+			`Resetting cache for ${this.oid} with ${operations.length} ops and ${baselines.length} baselines, dropUnconfirmed=${dropUnconfirmed}`,
 		);
-		if (dropUnconfirmed) {
-			this.operationsMap = new Map();
-		} else {
-			// clear out all confirmed operations, leaving only unconfirmed
-			// which have been added in memory but not yet persisted in storage
-			for (const oid of this.operationsMap.keys()) {
-				this.operationsMap.set(
-					oid,
-					this.operationsMap.get(oid)?.filter((op) => !op.confirmed) ?? [],
-				);
+		const info = { isLocal: false, affectedOids: new Set<ObjectIdentifier>() };
+
+		// NOTE: not clearing these maps... there are even more
+		// race conditions where we begin opening a cache, queue up a
+		// reset, then receive incoming operations before the reset
+		// actually hits this function, so those incoming ops are
+		// dropped.
+		// FIXME: include this in a future refactor of this
+		// whole system.
+
+		if (dropAll) this.baselinesMap.clear();
+		this.insertBaselines(baselines, info);
+
+		if (dropAll) this.operationsMap.clear();
+		this.insertOperations(operations, info);
+
+		if (unconfirmedOperations || dropUnconfirmed) {
+			this.localOperationsMap.clear();
+			if (unconfirmedOperations) {
+				this.insertLocalOperations(unconfirmedOperations);
 			}
 		}
-		this.insertOperations(operations, info);
+
 		for (const oid of this.entities.keys()) {
 			const entityRef = this.entities.get(oid);
 			const entity = entityRef?.deref();

package/src/entities/Entity.ts

@@ -6,6 +6,7 @@ import {
 	decomposeOid,
 	EventSubscriber,
 	FileData,
+	FileRef,
 	isFileRef,
 	isObjectRef,
 	maybeGetOid,
@@ -19,7 +20,6 @@ import {
 } from '@verdant-web/common';
 import { EntityFile } from '../files/EntityFile.js';
 import { processValueFiles } from '../files/utils.js';
-import { WeakRef } from './FakeWeakRef.js';
 
 export const ADD_OPERATIONS = '@@addOperations';
 export const DELETE = '@@delete';
@@ -39,6 +39,7 @@ export interface CacheTools {
 		parent?: Entity,
 	): Entity;
 	hasOid(oid: ObjectIdentifier): boolean;
+	weakRef<T extends object>(value: T): WeakRef<T>;
 }
 
 export interface StoreTools {
@@ -103,10 +104,11 @@ type EntityEvents = {
 type BaseEntityValue = { [Key: string]: any } | any[];
 
 export class Entity<
-	Init = any,
-	KeyValue extends BaseEntityValue = any,
-	Snapshot extends any = DataFromInit<Init>,
-> implements
+		Init = any,
+		KeyValue extends BaseEntityValue = any,
+		Snapshot extends any = DataFromInit<Init>,
+	>
+	implements
 		ObjectEntity<Init, KeyValue, Snapshot>,
 		ListEntity<Init, KeyValue, Snapshot>
 {
@@ -120,6 +122,7 @@ export class Entity<
 	protected readonly cache: CacheTools;
 	protected _deleted = false;
 	protected parent: WeakRef<Entity<any, any>> | undefined;
+	protected readonly readonlyKeys: (keyof Init)[];
 
 	private cachedSnapshot: any = null;
 	private cachedDestructure: KeyValue | null = null;
@@ -205,6 +208,7 @@ export class Entity<
 		cache,
 		parent,
 		onAllUnsubscribed,
+		readonlyKeys = [],
 	}: {
 		oid: ObjectIdentifier;
 		store: StoreTools;
@@ -212,14 +216,16 @@ export class Entity<
 		cache: CacheTools;
 		parent?: Entity<any, any>;
 		onAllUnsubscribed?: () => void;
+		readonlyKeys?: (keyof Init)[];
 	}) {
 		this.oid = oid;
 		const { collection } = decomposeOid(oid);
 		this.collection = collection;
-		this.parent = parent && new WeakRef(parent);
 		this.store = store;
 		this.fieldSchema = fieldSchema;
+		this.readonlyKeys = readonlyKeys;
 		this.cache = cache;
+		this.parent = parent && this.cache.weakRef(parent);
 		const { view, deleted, lastTimestamp } = this.cache.computeView(oid);
 		this._current = view;
 		this._deleted = deleted;
@@ -402,6 +408,17 @@ export class Entity<
 		return result;
 	};
 
+	private getFileSnapshot(item: FileRef) {
+		const file = this.store.getFile(item.id);
+		if (file.url) {
+			return { id: item.id, url: file.url };
+		} else if (file.loading || file.failed) {
+			return { id: item.id, url: undefined };
+		} else {
+			return { id: item.id, url: null };
+		}
+	}
+
 	/**
 	 * Returns a copy of the entity and all sub-objects as
 	 * a plain object or array.
@@ -421,6 +438,8 @@ export class Entity<
 			snapshot = this.value.map((item, idx) => {
 				if (isObjectRef(item)) {
 					return this.getSubObject(item.id, idx)?.getSnapshot();
+				} else if (isFileRef(item)) {
+					return this.getFileSnapshot(item);
 				}
 				return item;
 			}) as Snapshot;
@@ -429,6 +448,8 @@ export class Entity<
 			for (const [key, value] of Object.entries(snapshot)) {
 				if (isObjectRef(value)) {
 					snapshot[key] = this.getSubObject(value.id, key)?.getSnapshot();
+				} else if (isFileRef(value)) {
+					snapshot[key] = this.getFileSnapshot(value);
 				}
 			}
 		}
@@ -451,6 +472,9 @@ export class Entity<
 		return Object.values(this.getAll());
 	};
 	set = <Key extends keyof Init>(key: Key, value: Init[Key]) => {
+		if (this.readonlyKeys.includes(key)) {
+			throw new Error(`Cannot set readonly key ${key.toString()}`);
+		}
 		this.addPatches(
 			this.store.patchCreator.createSet(
 				this.oid,
@@ -480,6 +504,9 @@ export class Entity<
 		}
 	};
 	private getDeleteMode = (key: any) => {
+		if (this.readonlyKeys.includes(key)) {
+			return false;
+		}
 		// 'any' is always deletable, and map values can be removed completely
 		if (this.fieldSchema.type === 'any' || this.fieldSchema.type === 'map') {
 			return 'delete';
@@ -532,6 +559,9 @@ export class Entity<
 			);
 		}
 		for (const [key, field] of Object.entries(value)) {
+			if (this.readonlyKeys.includes(key as any)) {
+				throw new Error(`Cannot set readonly key ${key.toString()}`);
+			}
 			const fieldSchema = this.getChildFieldSchema(key);
 			if (fieldSchema) {
 				traverseCollectionFieldsAndApplyDefaults(field, fieldSchema);
@@ -718,6 +748,10 @@ export class Entity<
 	find = (predicate: (value: ListItemValue<KeyValue>) => boolean) => {
 		return this.getAsWrapped().find(predicate);
 	};
+
+	includes = (item: ListItemValue<KeyValue>) => {
+		return this.has(item);
+	};
 }
 
 export interface BaseEntity<
@@ -786,6 +820,7 @@ export interface ListEntity<
 	find(
 		predicate: (value: ListItemValue<Value>) => boolean,
 	): ListItemValue<Value> | undefined;
+	includes(value: ListItemValue<Value>): boolean;
 }
 
 export type AnyEntity<