@verdant-web/store 2.0.1

package/src/client/Client.ts

@@ -0,0 +1,307 @@
+import {
+	assert,
+	DocumentBaseline,
+	Migration,
+	Operation,
+	SchemaCollection,
+} from '@verdant-web/common';
+import { Context } from '../context.js';
+import { DocumentManager } from '../DocumentManager.js';
+import { FileManager, FileManagerConfig } from '../files/FileManager.js';
+import { closeDatabase, getSizeOfObjectStore } from '../idb.js';
+import { ExportData, Metadata } from '../metadata/Metadata.js';
+import { openDocumentDatabase } from '../openDocumentDatabase.js';
+import { EntityStore } from '../reactives/EntityStore.js';
+import { NoSync, ServerSync, ServerSyncOptions, Sync } from '../sync/Sync.js';
+import { CollectionQueries } from '../queries2/CollectionQueries.js';
+import { QueryCache } from '../queries2/QueryCache.js';
+
+interface ClientConfig<Presence = any> {
+	syncConfig?: ServerSyncOptions<Presence>;
+	migrations: Migration[];
+	files?: FileManagerConfig;
+}
+
+// not actually used below, but helpful for internal code which
+// might rely on this stuff...
+export type ClientWithCollections = Client & {
+	[key: string]: CollectionQueries<any, any, any>;
+};
+
+export class Client {
+	readonly meta: Metadata;
+	private _entities: EntityStore;
+	private _queryCache: QueryCache;
+	private _documentManager: DocumentManager<any>;
+	private _fileManager: FileManager;
+
+	readonly collectionNames: string[];
+
+	private _sync!: Sync;
+
+	get sync() {
+		return this._sync;
+	}
+
+	get entities() {
+		return this._entities;
+	}
+
+	get documentManager() {
+		return this._documentManager;
+	}
+
+	constructor(
+		private config: ClientConfig,
+		private context: Context,
+		components: { meta: Metadata },
+	) {
+		this.meta = components.meta;
+		this.collectionNames = Object.keys(context.schema.collections);
+		this._sync = this.config.syncConfig
+			? new ServerSync(this.config.syncConfig, {
+					meta: this.meta,
+					onData: this.addData,
+					log: this.context.log,
+			  })
+			: new NoSync();
+
+		this._fileManager = new FileManager({
+			db: this.metaDb,
+			sync: this.sync,
+			context: this.context,
+			config: this.config.files,
+			meta: this.meta,
+		});
+		this._entities = new EntityStore({
+			context: this.context,
+			meta: this.meta,
+			files: this._fileManager,
+		});
+		this._queryCache = new QueryCache({
+			context,
+		});
+		this._documentManager = new DocumentManager(
+			this.meta,
+			this.schema,
+			this._entities,
+		);
+
+		this.documentDb.addEventListener('versionchange', () => {
+			this.context.log?.(
+				`Another tab has requested a version change for ${this.namespace}`,
+			);
+			this.documentDb.close();
+			if (typeof window !== 'undefined') {
+				window.location.reload();
+			}
+		});
+
+		this.metaDb.addEventListener('versionchange', () => {
+			this.context.log?.(
+				`Another tab has requested a version change for ${this.namespace}`,
+			);
+			this.metaDb.close();
+			if (typeof window !== 'undefined') {
+				window.location.reload();
+			}
+		});
+
+		// self-assign collection shortcuts. these are not typed
+		// here but are typed in the generated code...
+		for (const [name, _collection] of Object.entries(
+			context.schema.collections,
+		)) {
+			const collection = _collection as SchemaCollection<any, any>;
+			const collectionName = collection.pluralName ?? collection.name + 's';
+			// TODO: untangle this requirement
+			assert(
+				collectionName === name,
+				`The key of the collection in the schema must be the plural of the name (expected: "${collectionName}")`,
+			);
+			(this as any)[collectionName] = new CollectionQueries({
+				collection: collectionName,
+				cache: this._queryCache,
+				context: this.context,
+				entities: this.entities,
+				documentManager: this.documentManager,
+			});
+		}
+	}
+
+	private addData = (data: {
+		operations: Operation[];
+		baselines: DocumentBaseline[];
+		reset?: boolean;
+	}) => {
+		return this._entities.addData(data);
+	};
+
+	get documentDb() {
+		return this.context.documentDb;
+	}
+
+	get metaDb() {
+		return this.context.metaDb;
+	}
+
+	get schema() {
+		return this.context.schema;
+	}
+
+	get namespace() {
+		return this.context.namespace;
+	}
+
+	get undoHistory() {
+		return this.context.undoHistory;
+	}
+
+	/**
+	 * @deprecated - use client.sync.presence instead
+	 */
+	get presence() {
+		return this.sync.presence;
+	}
+
+	/**
+	 * Batch multiple operations together to be executed in a single transaction.
+	 * The changes made will not be included in the same undo history step as
+	 * any other changes made outside of the batch. You can also disable undo
+	 * for your batch to omit changes from undo history.
+	 *
+	 * Provide a batch name to apply multiple changes to the same batch
+	 * across different invocations. Batches will automatically flush after
+	 * a short delay or if they reach a maximum size.
+	 */
+	get batch() {
+		return this.entities.batch;
+	}
+
+	stats = async () => {
+		const collectionNames = Object.keys(this.schema.collections);
+		let collections = {} as Record<string, { count: number; size: number }>;
+		for (const collectionName of collectionNames) {
+			collections[collectionName] = await getSizeOfObjectStore(
+				this.documentDb,
+				collectionName,
+			);
+		}
+		const meta = await this.meta.stats();
+		const storage =
+			typeof navigator !== 'undefined' &&
+			typeof navigator.storage !== 'undefined' &&
+			'estimate' in navigator.storage
+				? await navigator.storage.estimate()
+				: undefined;
+
+		// determine data:metadata ratio for total size of all collections vs metadata
+		const totalCollectionsSize = Object.values(collections).reduce(
+			(acc, { size }) => acc + size,
+			0,
+		);
+		const totalMetaSize = meta.baselinesSize.size + meta.operationsSize.size;
+		const metaToDataRatio = totalMetaSize / totalCollectionsSize;
+
+		return {
+			collections,
+			meta,
+			storage,
+			totalMetaSize,
+			totalCollectionsSize,
+			metaToDataRatio,
+			quotaUsage:
+				storage?.usage && storage?.quota
+					? storage.usage / storage.quota
+					: undefined,
+		};
+	};
+
+	close = async () => {
+		await this.entities.flushPatches();
+		this.sync.stop();
+		this.sync.dispose();
+
+		this.meta.close();
+
+		this._entities.destroy();
+
+		await closeDatabase(this.documentDb);
+		await closeDatabase(this.metaDb);
+
+		this.context.log?.('Client closed');
+	};
+
+	__dangerous__resetLocal = async () => {
+		this.sync.stop();
+		const req1 = indexedDB.deleteDatabase([this.namespace, 'meta'].join('_'));
+		const req2 = indexedDB.deleteDatabase(
+			[this.namespace, 'collections'].join('_'),
+		);
+		await Promise.all([
+			new Promise((resolve, reject) => {
+				req1.onsuccess = resolve;
+				req1.onerror = reject;
+			}),
+			new Promise((resolve, reject) => {
+				req2.onsuccess = resolve;
+				req2.onerror = reject;
+			}),
+		]);
+		window.location.reload();
+	};
+
+	export = async () => {
+		const metaExport = await this.meta.export();
+		return Buffer.from(JSON.stringify(metaExport));
+	};
+
+	import = async (buffer: Buffer) => {
+		this.context.log('Importing data...');
+		// close the document DB
+		await closeDatabase(this.context.documentDb);
+
+		const metaExport = JSON.parse(buffer.toString()) as ExportData;
+		await this.meta.resetFrom(metaExport);
+		// now reset the document DB to the specified version
+		// and run migrations to get it to the latest version
+		const version = metaExport.schema.version;
+		const deleteReq = indexedDB.deleteDatabase(
+			[this.namespace, 'collections'].join('_'),
+		);
+		await new Promise((resolve, reject) => {
+			deleteReq.onsuccess = resolve;
+			deleteReq.onerror = reject;
+		});
+		// reset our context to the imported schema for now
+		const currentSchema = this.context.schema;
+		this.context.schema = metaExport.schema;
+		// now open the document DB empty at the specified version
+		// and initialize it from the meta DB
+		this.context.documentDb = await openDocumentDatabase({
+			meta: this.meta,
+			migrations: this.config.migrations,
+			context: this.context,
+			version,
+		});
+		// re-initialize data
+		this.context.log('Re-initializing data from imported data...');
+		await this._entities.addData({
+			operations: metaExport.operations,
+			baselines: metaExport.baselines,
+			reset: true,
+		});
+		// close the database and reopen to latest version, applying
+		// migrations
+		await closeDatabase(this.context.documentDb);
+		this.context.log('Migrating up to latest schema...');
+		// put the schema back
+		this.context.schema = currentSchema;
+		this.context.documentDb = await openDocumentDatabase({
+			meta: this.meta,
+			migrations: this.config.migrations,
+			context: this.context,
+			version: currentSchema.version,
+		});
+	};
+}

package/src/client/ClientDescriptor.ts

@@ -0,0 +1,172 @@
+import { EventSubscriber, Migration, StorageSchema } from '@verdant-web/common';
+import { Context } from '../context.js';
+import { FileManagerConfig } from '../files/FileManager.js';
+import { ReturnedFileData } from '../files/FileStorage.js';
+import { Metadata } from '../metadata/Metadata.js';
+import { openMetadataDatabase } from '../metadata/openMetadataDatabase.js';
+import { openDocumentDatabase } from '../openDocumentDatabase.js';
+import { ServerSyncOptions } from '../sync/Sync.js';
+import { UndoHistory } from '../UndoHistory.js';
+import { Client } from './Client.js';
+
+export interface ClientDescriptorOptions<Presence = any, Profile = any> {
+	/** The schema used to create this client */
+	schema: StorageSchema<any>;
+	/** Migrations, in order, to upgrade to each successive version of the schema */
+	migrations: Migration<any>[];
+	/** Provide a sync config to turn on synchronization with a server */
+	sync?: ServerSyncOptions<Profile, Presence>;
+	/** Optionally override the IndexedDB implementation */
+	indexedDb?: IDBFactory;
+	/**
+	 * Namespaces are used to separate data from different clients in IndexedDB.
+	 */
+	namespace: string;
+	/**
+	 * Provide your own UndoHistory to have a unified undo system across multiple
+	 * clients if you so desire.
+	 */
+	undoHistory?: UndoHistory;
+	/**
+	 * Provide a log function to log internal debug messages
+	 */
+	log?: (...args: any[]) => void;
+	disableRebasing?: boolean;
+	/**
+	 * Provide a specific schema number to override the schema version
+	 * in the database. This is useful for testing migrations or recovering
+	 * from a mistakenly deployed incorrect schema. A specific version is required
+	 * so that you don't leave this on accidentally for all new schemas.
+	 */
+	overrideSchemaConflict?: number;
+	/**
+	 * Configuration for file management
+	 */
+	files?: FileManagerConfig;
+}
+
+/**
+ * Since storage initialization is async, this class wraps the core
+ * Storage creation promise and exposes some metadata which can
+ * be useful immediately.
+ */
+export class ClientDescriptor<Presence = any, Profile = any> {
+	private readonly _readyPromise: Promise<Client>;
+	// assertions because these are defined by plucking them from
+	// Promise initializer
+	private resolveReady!: (storage: Client) => void;
+	private rejectReady!: (err: Error) => void;
+	private _resolvedValue: Client | undefined;
+	private _initializing = false;
+	private _namespace: string;
+
+	get namespace() {
+		return this._namespace;
+	}
+
+	constructor(
+		private readonly init: ClientDescriptorOptions<Presence, Profile>,
+	) {
+		this._readyPromise = new Promise((resolve, reject) => {
+			this.resolveReady = resolve;
+			this.rejectReady = reject;
+		});
+		this._namespace = init.namespace;
+	}
+
+	private initialize = async (init: ClientDescriptorOptions) => {
+		// if server-side and no alternative IndexedDB implementation was provided,
+		// we can't initialize the storage
+		if (!('indexedDB' in window) && !init.indexedDb) {
+			throw new Error(
+				'A verdant client was initialized in an environment without IndexedDB. If you are using verdant in a server-rendered framework, you must enforce that all clients are initialized on the client-side, or you must provide some mock interface of IDBFactory to the ClientDescriptor options.',
+			);
+		}
+
+		if (this._initializing || this._resolvedValue) {
+			return this._readyPromise;
+		}
+		this._initializing = true;
+		try {
+			const metaDbName = [init.namespace, 'meta'].join('_');
+			const { db: metaDb } = await openMetadataDatabase(this._namespace, {
+				indexedDB: init.indexedDb,
+				log: init.log,
+				databaseName: metaDbName,
+			});
+
+			const context: Omit<Context, 'documentDb'> = {
+				namespace: this._namespace,
+				metaDb,
+				schema: init.schema,
+				log: init.log || (() => {}),
+				undoHistory: init.undoHistory || new UndoHistory(),
+				entityEvents: new EventSubscriber(),
+			};
+			const meta = new Metadata({
+				context,
+				disableRebasing: init.disableRebasing,
+			});
+
+			// verify schema integrity
+			await meta.updateSchema(init.schema, init.overrideSchemaConflict);
+
+			const documentDb = await openDocumentDatabase({
+				context,
+				version: init.schema.version,
+				meta,
+				migrations: init.migrations,
+				indexedDB: init.indexedDb,
+			});
+
+			const fullContext: Context = Object.assign(context, { documentDb });
+
+			const storage = new Client(
+				{
+					syncConfig: init.sync,
+					migrations: init.migrations,
+					files: init.files,
+				},
+				fullContext,
+				{
+					meta,
+				},
+			);
+
+			this.resolveReady(storage);
+			this._resolvedValue = storage;
+			return storage;
+		} catch (err) {
+			this.rejectReady(err as Error);
+			throw err;
+		} finally {
+			this._initializing = false;
+		}
+	};
+
+	get current() {
+		// exposing an immediate value if already resolved lets us
+		// skip the promise microtask when accessing this externally if
+		// the initialization has been completed.
+		return this._resolvedValue;
+	}
+
+	get readyPromise() {
+		return this._readyPromise;
+	}
+
+	get schema() {
+		return this.init.schema;
+	}
+
+	open = () => this.initialize(this.init);
+
+	close = async () => {
+		if (this._resolvedValue) {
+			this._resolvedValue.close();
+		}
+		if (this._initializing) {
+			(await this._readyPromise).close();
+		}
+	};
+}

package/src/constants.ts

@@ -0,0 +1 @@
+export const TEST_API = '__unsafe__test_api__';

package/src/context.ts

@@ -0,0 +1,18 @@
+import { EventSubscriber, StorageSchema } from '@verdant-web/common';
+import { UndoHistory } from './UndoHistory.js';
+
+/**
+ * Common components utilized across various client
+ * services.
+ */
+export interface Context {
+	namespace: string;
+	metaDb: IDBDatabase;
+	documentDb: IDBDatabase;
+	undoHistory: UndoHistory;
+	schema: StorageSchema;
+	log: (...args: any[]) => void;
+	entityEvents: EventSubscriber<{
+		collectionsChanged: (names: string[]) => void;
+	}>;
+}

package/src/files/EntityFile.ts

@@ -0,0 +1,74 @@
+import { EventSubscriber, FileData } from '@verdant-web/common';
+
+export type EntityFileEvents = {
+	change: () => void;
+};
+
+export const UPDATE = Symbol('entity-file-update');
+export const MARK_FAILED = Symbol('entity-file-mark-failed');
+
+/**
+ * Provides a consistent interface for files used in an app via
+ * Entity access.
+ */
+export class EntityFile extends EventSubscriber<EntityFileEvents> {
+	// cached object URL for a local blob file, if applicable
+	private _objectUrl: string | null = null;
+	private _fileData: FileData | null = null;
+	private _loading = true;
+	private _failed = false;
+	private _disposed = false;
+
+	constructor(public readonly id: string) {
+		super();
+	}
+
+	[UPDATE] = (fileData: FileData) => {
+		this._loading = false;
+		this._failed = false;
+		this._disposed = false;
+		this._fileData = fileData;
+		if (fileData.file) {
+			if (this._objectUrl) {
+				URL.revokeObjectURL(this._objectUrl);
+			}
+			this._objectUrl = URL.createObjectURL(fileData.file);
+		}
+		this.emit('change');
+	};
+
+	[MARK_FAILED] = () => {
+		this._failed = true;
+		this._loading = false;
+		this.emit('change');
+	};
+
+	get url(): string | null {
+		if (this.loading) return null;
+		if (this._objectUrl) return this._objectUrl;
+		return this._fileData?.url ?? null;
+	}
+
+	get name(): string | null {
+		return this._fileData?.name ?? null;
+	}
+
+	get type(): string | null {
+		return this._fileData?.type ?? null;
+	}
+
+	get loading() {
+		return this._loading;
+	}
+
+	get failed() {
+		return this._failed;
+	}
+
+	dispose = () => {
+		if (this._objectUrl) {
+			URL.revokeObjectURL(this._objectUrl);
+		}
+		this._disposed = true;
+	};
+}