@verdant-web/store 4.6.1 → 5.0.1

package/src/context/context.ts

@@ -3,30 +3,117 @@ import {
 	EventSubscriber,
 	FileData,
 	FileRef,
+	HybridLogicalClockTimestampProvider,
 	Migration,
 	ObjectIdentifier,
 	Operation,
 	PatchCreator,
 	StorageSchema,
+	VerdantError,
 } from '@verdant-web/common';
+import { FakeWeakRef } from '../FakeWeakRef.js';
 import { UndoHistory } from '../UndoHistory.js';
 import type { Client } from '../client/Client.js';
-import { VerdantLogger } from '../logger.js';
+import { debugLogger, noLogger, VerdantLogger } from '../logger.js';
 import { PersistenceFiles } from '../persistence/PersistenceFiles.js';
 import type { PersistenceMetadata } from '../persistence/PersistenceMetadata.js';
 import type { PersistenceDocuments } from '../persistence/PersistenceQueries.js';
+import { IdbPersistence } from '../persistence/idb/idbPersistence.js';
 import {
 	PersistedFileData,
 	PersistenceImplementation,
 } from '../persistence/interfaces.js';
+import { initializePersistence } from '../persistence/persistence.js';
+import { ServerSyncOptions } from '../sync/Sync.js';
 import { ShutdownHandler } from './ShutdownHandler.js';
 import { Time } from './Time.js';
 
+export interface ContextEnvironment {
+	WebSocket: typeof WebSocket;
+	fetch: typeof fetch;
+	indexedDB: typeof indexedDB;
+	location: Location;
+	history: History;
+}
+
+export const defaultBrowserEnvironment: ContextEnvironment = {
+	WebSocket: typeof WebSocket !== 'undefined' ? WebSocket : (undefined as any),
+	fetch: typeof window !== 'undefined' ? window.fetch.bind(window) : fetch!,
+	indexedDB: typeof indexedDB !== 'undefined' ? indexedDB : (undefined as any),
+	location:
+		typeof window !== 'undefined' ? window.location : (undefined as any),
+	history: typeof window !== 'undefined' ? window.history : (undefined as any),
+};
+
+export interface ContextInit {
+	/** The schema used to create this client */
+	schema: StorageSchema<any>;
+	oldSchemas: StorageSchema<any>[];
+	/** Migrations, in order, to upgrade to each successive version of the schema */
+	migrations: Migration<any>[];
+	/** Provide a sync config to turn on synchronization with a server */
+	sync?: ServerSyncOptions;
+	/**
+	 * Namespaces are used to separate data from different clients in IndexedDB.
+	 */
+	namespace: string;
+	/**
+	 * Provide your own UndoHistory to have a unified undo system across multiple
+	 * clients if you so desire.
+	 */
+	undoHistory?: UndoHistory;
+	/**
+	 * Provide a log function to log internal debug messages
+	 */
+	log?: VerdantLogger | false;
+	disableRebasing?: boolean;
+	rebaseTimeout?: number;
+	/**
+	 * Provide a specific schema number to override the schema version
+	 * in the database. This is useful for testing migrations or recovering
+	 * from a mistakenly deployed incorrect schema. A specific version is required
+	 * so that you don't leave this on accidentally for all new schemas.
+	 */
+	overrideSchemaConflict?: number;
+	/**
+	 * Configuration for file management
+	 */
+	files?: FileConfig;
+
+	/**
+	 * Override the default IndexedDB persistence implementation.
+	 */
+	persistence?: PersistenceImplementation;
+
+	/**
+	 * Specify the environment dependencies needed for the client.
+	 * Normally these are provided by the browser, but in other
+	 * runtimes you may need to provide your own.
+	 */
+	environment?: Partial<ContextEnvironment>;
+
+	/**
+	 * Enables experimental WeakRef usage to cull documents
+	 * from cache that aren't being used. This is a performance
+	 * optimization which has been tested under all Verdant's test
+	 * suites but I still want to keep testing it in the real world
+	 * before turning it on.
+	 */
+	EXPERIMENTAL_weakRefs?: boolean;
+
+	/**
+	 * Customize querying behavior.
+	 */
+	queries?: QueryConfig;
+
+	persistenceShutdownHandler?: ShutdownHandler;
+}
+
 /**
  * Common components utilized across various client
  * services.
  */
-export interface Context {
+export class Context {
 	namespace: string;
 	/**
 	 * when in WIP mode, namespace might be set to a temporary value. This will always point to the
@@ -35,9 +122,16 @@ export interface Context {
 	originalNamespace: string;
 	time: Time;
 
-	meta: PersistenceMetadata;
-	documents: PersistenceDocuments;
-	files: PersistenceFiles;
+	// async initialized services
+	get meta(): Promise<PersistenceMetadata> {
+		return this.initializedPromise.then((init) => init.meta);
+	}
+	get documents(): Promise<PersistenceDocuments> {
+		return this.initializedPromise.then((init) => init.documents);
+	}
+	get files(): Promise<PersistenceFiles> {
+		return this.initializedPromise.then((init) => init.files);
+	}
 
 	undoHistory: UndoHistory;
 	schema: StorageSchema;
@@ -57,6 +151,7 @@ export interface Context {
 		fileAdded: (file: FileData) => void;
 		[ev: `fileUploaded:${string}`]: (file: FileData) => void;
 		fileUploaded: (file: FileData) => void;
+		outgoingSyncMessage: (message: ClientMessage) => void;
 	}>;
 	globalEvents: EventSubscriber<{
 		/**
@@ -86,15 +181,22 @@ export interface Context {
 		rebase: () => void;
 		fileSaved: (file: FileData) => void;
 	}>;
-	weakRef<T extends object>(value: T): WeakRef<T>;
+	weakRef = <T extends object>(value: T): WeakRef<T> => {
+		if (this.init.EXPERIMENTAL_weakRefs) {
+			return new WeakRef(value);
+		}
+		return new FakeWeakRef(value) as any;
+	};
 	migrations: Migration<any>[];
-	closing: boolean;
 	/** If this is present, any attempt to close the client should await it first. */
 	closeLock?: Promise<void>;
-	pauseRebasing: boolean;
 	patchCreator: PatchCreator;
 	persistenceShutdownHandler: ShutdownHandler;
 
+	// state
+	closing: boolean = false;
+	pauseRebasing: boolean = false;
+
 	config: {
 		files?: FileConfig;
 		sync?: SyncConfig;
@@ -102,13 +204,7 @@ export interface Context {
 		queries?: QueryConfig;
 	};
 
-	environment: {
-		WebSocket: typeof WebSocket;
-		fetch: typeof fetch;
-		indexedDB: typeof indexedDB;
-		location: Location;
-		history: History;
-	};
+	environment: ContextEnvironment;
 
 	persistence: PersistenceImplementation;
 
@@ -116,7 +212,80 @@ export interface Context {
 	 * Must be defined by the Client once it exists. Attempts to use this before
 	 * it's ready will rightfully throw an error.
 	 */
-	getClient: () => Client;
+	getClient = (): Client => {
+		throw new VerdantError(
+			VerdantError.Code.Unexpected,
+			undefined,
+			'Client not yet initialized. This is a Verdant bug, please report it.',
+		);
+	};
+
+	constructor(
+		private init: ContextInit,
+		initPersistence?: ReturnType<typeof initializePersistence>,
+	) {
+		// if server-side and no alternative IndexedDB implementation was provided,
+		// we can't initialize the storage
+		if (typeof window === 'undefined' && !this.init.environment) {
+			throw new Error(
+				'A Verdant client was initialized in an environment without a global Window or `environment` configuration. If you are using verdant in a server-rendered framework, you must enforce that all clients are initialized on the client-side, or you must provide some mock interface of the environment to the Client options.',
+			);
+		}
+		// set static values
+		this.namespace = this.init.namespace;
+		this.originalNamespace = this.init.namespace;
+		this.time = new Time(
+			new HybridLogicalClockTimestampProvider(),
+			this.init.schema.version,
+		);
+		this.log =
+			this.init.log === false ? noLogger : this.init.log || debugLogger('🌿');
+		this.migrations = init.migrations;
+		this.undoHistory = init.undoHistory || new UndoHistory();
+		this.entityEvents = new EventSubscriber();
+		this.internalEvents = new EventSubscriber();
+		this.globalEvents = new EventSubscriber();
+		this.schema = init.schema;
+		this.oldSchemas = init.oldSchemas;
+		this.patchCreator = new PatchCreator(() => this.time.now);
+		this.persistenceShutdownHandler =
+			init.persistenceShutdownHandler || new ShutdownHandler(this.log);
+		this.config = {
+			files: init.files,
+			sync: init.sync,
+			persistence: {
+				disableRebasing: init.disableRebasing,
+				rebaseTimeout: init.rebaseTimeout,
+			},
+			queries: init.queries,
+		};
+		this.environment = {
+			...defaultBrowserEnvironment,
+			...init.environment,
+		};
+		this.persistence =
+			init.persistence || new IdbPersistence(this.environment.indexedDB);
+		this.initializedPromise = initPersistence || initializePersistence(this);
+		this.initializedPromise.then(() => {
+			this.log('info', 'Persistence initialized');
+		});
+	}
+	private initializedPromise;
+	get waitForInitialization(): Promise<void> {
+		return this.initializedPromise.then(() => {});
+	}
+	reinitialize = async (): Promise<void> => {
+		this.initializedPromise = initializePersistence(this);
+		await this.initializedPromise;
+	};
+
+	cloneWithOptions(options: Partial<ContextInit>): Context {
+		const copy = new Context(
+			{ ...this.init, ...options },
+			this.initializedPromise,
+		);
+		return copy;
+	}
 }
 
 export interface FileConfig {
@@ -231,4 +400,7 @@ export interface QueryConfig {
 	evictionTime?: number;
 }
 
-export type InitialContext = Omit<Context, 'documents' | 'meta' | 'files'>;
+export type ContextWithoutPersistence = Omit<
+	Context,
+	'meta' | 'documents' | 'files'
+>;

package/src/entities/Entity.test.ts

@@ -261,7 +261,6 @@ describe('Entity', () => {
 			entity.update({ string: 'new world' });
 			expect(onPendingOperations).toHaveBeenCalledTimes(1);
 			const operation = onPendingOperations.mock.calls[0][0][0];
-			console.log(operation);
 
 			entity.__discardPendingOperation__(operation);
 

package/src/entities/EntityMetadata.ts

@@ -274,12 +274,23 @@ export class EntityMetadata {
 			if (op.data.op === 'delete') {
 				deleted = true;
 			} else {
-				base = applyPatch(base, op.data);
-				if (op.data.op === 'initialize') {
-					deleted = false;
-					if (op.authz) {
-						authz = op.authz;
+				try {
+					base = applyPatch(base, op.data);
+					if (op.data.op === 'initialize') {
+						deleted = false;
+						if (op.authz) {
+							authz = op.authz;
+						}
 					}
+				} catch (err) {
+					this.ctx.log(
+						'critical',
+						`Failed to apply operation to entity ${this.oid}: ${JSON.stringify(
+							op,
+						)}`,
+						err,
+					);
+					throw err;
 				}
 			}
 

package/src/entities/EntityStore.ts

@@ -134,7 +134,7 @@ export class EntityStore extends Disposable {
 	};
 
 	empty = async () => {
-		await this.ctx.documents.reset();
+		await (await this.ctx.documents).reset();
 		this.events.resetAll.invoke(this);
 		this.cache.clear();
 	};
@@ -144,8 +144,8 @@ export class EntityStore extends Disposable {
 			this.ctx.log('warn', 'EntityStore is disposed, not resetting local data');
 			return;
 		}
-		await this.ctx.meta.reset();
-		await this.ctx.documents.reset();
+		await (await this.ctx.meta).reset();
+		await (await this.ctx.documents).reset();
 		this.events.resetAll.invoke(this);
 	};
 
@@ -223,7 +223,7 @@ export class EntityStore extends Disposable {
 		// TODO: could messages be sent to sync before storage,
 		// so that realtime is lower latency? What would happen
 		// if the storage failed?
-		await this.ctx.meta.insertData(data, abortOptions);
+		await (await this.ctx.meta).insertData(data, abortOptions);
 		this.ctx.log(
 			'debug',
 			'Data processing complete, all data saved to metadata db.',
@@ -247,7 +247,7 @@ export class EntityStore extends Disposable {
 		);
 		try {
 			this.ctx.log('debug', 'Saving entities to queryable storage');
-			await this.ctx.documents.saveEntities(entities, abortOptions);
+			await (await this.ctx.documents).saveEntities(entities, abortOptions);
 		} catch (err) {
 			if (this.disposed) {
 				this.ctx.log(
@@ -319,6 +319,7 @@ export class EntityStore extends Disposable {
 	};
 
 	destroy = async () => {
+		this.ctx.log('warn', 'Disposing EntityStore');
 		this.dispose();
 		await this.batcher.flushAll();
 	};
@@ -469,7 +470,11 @@ export class EntityStore extends Disposable {
 		}
 
 		if (this.disposed) {
-			throw new Error('Cannot hydrate entity after store has been disposed');
+			this.ctx.log(
+				'warn',
+				'Cannot hydrate entity after store has been disposed',
+			);
+			return null;
 		}
 
 		const metadataFamily = new EntityFamilyMetadata({
@@ -522,10 +527,9 @@ export class EntityStore extends Disposable {
 		entity: Entity,
 		opts?: { abort: AbortSignal },
 	) => {
-		const { operations, baselines } = await this.ctx.meta.getDocumentData(
-			entity.oid,
-			opts,
-		);
+		const { operations, baselines } = await (
+			await this.ctx.meta
+		).getDocumentData(entity.oid, opts);
 
 		if (!baselines.length && !Object.keys(operations).length) {
 			this.ctx.log('debug', 'No data found for entity', entity.oid);

package/src/files/EntityFile.ts

@@ -138,7 +138,7 @@ export class EntityFile extends EventSubscriber<EntityFileEvents> {
 	getSnapshot(): FileData {
 		return {
 			id: this.id,
-			url: this._objectUrl ?? this._fileData?.url ?? undefined,
+			url: this._fileData?.url ?? this._objectUrl ?? undefined,
 			name: this.name ?? 'unknown-file',
 			remote: this._fileData?.remote ?? false,
 			type: this.type ?? '',

package/src/files/FileManager.ts

@@ -37,7 +37,7 @@ export class FileManager extends Disposable {
 		}
 		// this will download any original remote file and trigger a re-upload to the
 		// new file's identity, in addition to storing it on disk
-		const processedFile = await this.context.files.add(file);
+		const processedFile = await (await this.context.files).add(file);
 		entityFile[UPDATE](processedFile);
 	};
 
@@ -59,7 +59,7 @@ export class FileManager extends Disposable {
 	};
 
 	private load = async (file: EntityFile) => {
-		const fileData = await this.context.files.get(file.id);
+		const fileData = await (await this.context.files).get(file.id);
 		if (fileData) {
 			file[UPDATE](fileData);
 		} else {
@@ -84,7 +84,7 @@ export class FileManager extends Disposable {
 
 				const result = await this.sync.getFile(file.id);
 				if (result.success) {
-					await this.context.files.add(result.data);
+					await (await this.context.files).add(result.data);
 					file[UPDATE](result.data);
 				} else {
 					this.context.log('error', 'Failed to load file', result);
@@ -97,8 +97,8 @@ export class FileManager extends Disposable {
 		}
 	};
 
-	private onFileUploaded = (data: FileData) => {
+	private onFileUploaded = async (data: FileData) => {
 		this.context.log('debug', 'Marking file as uploaded', data.id);
-		this.context.files.onUploaded(data.id);
+		(await this.context.files).onUploaded(data.id);
 	};
 }

package/src/index.ts

@@ -1,10 +1,10 @@
 import { Client } from './client/Client.js';
-import {
-	ClientDescriptor,
-	ClientDescriptorOptions,
-} from './client/ClientDescriptor.js';
+import type { ContextInit } from './context/context.js';
 export type { ClientWithCollections } from './client/Client.js';
-export { Client, ClientDescriptor };
+export { Client, type ClientDescriptorOptions, type ClientInitOptions };
+type ClientInitOptions = ContextInit;
+/** @deprecated - use ClientInitOptions alias */
+type ClientDescriptorOptions = ClientInitOptions;
 // backward compat
 export { createMigration, schema } from '@verdant-web/common';
 export type {
@@ -54,8 +54,4 @@ export { ServerSync, type ServerSyncOptions } from './sync/Sync.js';
 export type { SyncTransportMode } from './sync/Sync.js';
 export { UndoHistory } from './UndoHistory.js';
 export * from './utils/id.js';
-export { Client as Storage, ClientDescriptor as StorageDescriptor };
-export type {
-	ClientDescriptorOptions,
-	ClientDescriptorOptions as StorageInitOptions,
-};
+export { Client as Storage };

package/src/internal.ts

@@ -1,27 +1,26 @@
-export type { Context, InitialContext } from './context/context.js';
 export {
-	decomposeOid,
+	assert,
+	createCompoundIndexValue,
+	createLowerBoundIndexValue,
 	createOid,
+	createUpperBoundIndexValue,
+	decomposeOid,
 	generateId,
-	isRangeIndexFilter,
+	getIndexValues,
 	isCompoundIndexFilter,
 	isDirectSynthetic,
 	isMatchIndexFilter,
+	isMultiValueIndex,
+	isRangeIndexFilter,
 	isSortIndexFilter,
 	isStartsWithIndexFilter,
-	isMultiValueIndex,
-	assert,
-	createCompoundIndexValue,
-	createLowerBoundIndexValue,
-	createUpperBoundIndexValue,
-	getIndexValues,
 } from '@verdant-web/common';
 export type {
 	CollectionCompoundIndexFilter,
+	CollectionIndexFilter,
 	MatchCollectionIndexFilter,
 	RangeCollectionIndexFilter,
-	CollectionIndexFilter,
 } from '@verdant-web/common';
+export type { Context, ContextWithoutPersistence } from './context/context.js';
 export * from './persistence/migration/paths.js';
-export * from './persistence/migration/types.js';
 export { Disposable } from './utils/Disposable.js';

package/src/persistence/PersistenceMetadata.ts

@@ -2,14 +2,14 @@ import {
 	applyPatch,
 	assert,
 	assignOid,
-	ClientMessage,
 	DocumentBaseline,
-	EventSubscriber,
 	getOidRoot,
 	ObjectIdentifier,
 	Operation,
 	substituteRefsWithObjects,
 } from '@verdant-web/common';
+import cuid from 'cuid';
+import { ContextWithoutPersistence } from '../context/context.js';
 import {
 	AbstractTransaction,
 	ClientOperation,
@@ -18,20 +18,18 @@ import {
 	MetadataExport,
 	PersistenceMetadataDb,
 } from './interfaces.js';
-import { InitialContext } from '../context/context.js';
-import { PersistenceRebaser } from './PersistenceRebaser.js';
 import { MessageCreator } from './MessageCreator.js';
-import cuid from 'cuid';
+import { PersistenceRebaser } from './PersistenceRebaser.js';
 
 export class PersistenceMetadata {
 	private rebaser: PersistenceRebaser;
 	/** Available to others, like sync... */
 	readonly messageCreator: MessageCreator;
-	readonly events = new EventSubscriber<{
-		syncMessage: (message: ClientMessage) => void;
-	}>();
 
-	constructor(private db: PersistenceMetadataDb, private ctx: InitialContext) {
+	constructor(
+		private db: PersistenceMetadataDb,
+		private ctx: ContextWithoutPersistence,
+	) {
 		this.rebaser = new PersistenceRebaser(db, this, ctx);
 		this.messageCreator = new MessageCreator(db, this, ctx);
 	}
@@ -82,7 +80,7 @@ export class PersistenceMetadata {
 			`Inserted ${operations.length} local operations; sending sync message`,
 		);
 		const message = await this.messageCreator.createOperation({ operations });
-		this.events.emit('syncMessage', message);
+		this.ctx.internalEvents.emit('outgoingSyncMessage', message);
 	};
 
 	private insertRemoteOperations = async (
@@ -500,7 +498,7 @@ export class PersistenceMetadata {
 		// can't ack timestamps from the future.
 		if (timestamp > this.ctx.time.now) return;
 
-		this.events.emit('syncMessage', {
+		this.ctx.internalEvents.emit('outgoingSyncMessage', {
 			type: 'ack',
 			replicaId: localReplicaInfo.id,
 			timestamp,

package/src/persistence/idb/IdbService.ts

@@ -1,4 +1,4 @@
-import { Context, InitialContext } from '../../context/context.js';
+import { Context } from '../../context/context.js';
 import { Disposable } from '../../utils/Disposable.js';
 import {
 	createAbortableTransaction,
@@ -13,11 +13,10 @@ export class IdbService extends Disposable {
 
 	constructor(
 		protected db: IDBDatabase,
-		protected readonly ctx: InitialContext,
+		protected readonly ctx: Pick<Context, 'log' | 'environment'>,
 	) {
 		super();
 		const abortController = new AbortController();
-		const abort = abortController.abort.bind(abortController);
 		this.globalAbortController = abortController;
 		// FIXME: replace with event? I can't get this to work.
 		// this.addDispose(abort);