@verdant-web/store 3.12.1 → 4.0.0-next.1

package/src/client/ClientDescriptor.ts

@@ -1,34 +1,27 @@
 import {
 	EventSubscriber,
+	HybridLogicalClockTimestampProvider,
 	Migration,
-	Operation,
+	PatchCreator,
 	StorageSchema,
-	hashObject,
+	noop,
 } from '@verdant-web/common';
-import { Context } from '../context.js';
-import { FileManagerConfig } from '../files/FileManager.js';
-import { Metadata } from '../metadata/Metadata.js';
-import {
-	openMetadataDatabase,
-	openWIPMetadataDatabase,
-} from '../metadata/openMetadataDatabase.js';
-import { openWIPDatabase } from '../migration/openWIPDatabase.js';
+import { FileConfig, InitialContext } from '../context/context.js';
 import { ServerSyncOptions } from '../sync/Sync.js';
 import { UndoHistory } from '../UndoHistory.js';
 import { Client } from './Client.js';
-import {
-	deleteAllDatabases,
-	deleteDatabase,
-	getAllDatabaseNamesAndVersions,
-} from '../idb.js';
+import { deleteAllDatabases } from '../persistence/idb/util.js';
 import { FakeWeakRef } from '../FakeWeakRef.js';
 import { METADATA_VERSION_KEY } from './constants.js';
-import { openQueryDatabase } from '../migration/openQueryDatabase.js';
+import { Time } from '../context/Time.js';
+import { initializePersistence } from '../persistence/persistence.js';
+import { PersistenceImplementation } from '../persistence/interfaces.js';
+import { IdbPersistence } from '../persistence/idb/idbPersistence.js';
 
 export interface ClientDescriptorOptions<Presence = any, Profile = any> {
 	/** The schema used to create this client */
 	schema: StorageSchema<any>;
-	oldSchemas?: StorageSchema<any>[];
+	oldSchemas: StorageSchema<any>[];
 	/** Migrations, in order, to upgrade to each successive version of the schema */
 	migrations: Migration<any>[];
 	/** Provide a sync config to turn on synchronization with a server */
@@ -52,6 +45,7 @@ export interface ClientDescriptorOptions<Presence = any, Profile = any> {
 		...args: any[]
 	) => void;
 	disableRebasing?: boolean;
+	rebaseTimeout?: number;
 	/**
 	 * Provide a specific schema number to override the schema version
 	 * in the database. This is useful for testing migrations or recovering
@@ -62,14 +56,13 @@ export interface ClientDescriptorOptions<Presence = any, Profile = any> {
 	/**
 	 * Configuration for file management
 	 */
-	files?: FileManagerConfig;
+	files?: FileConfig;
 
 	/**
-	 * Listen for operations as they are applied to the database.
-	 * Wouldn't recommend using this unless you know what you're doing.
-	 * It's a very hot code path...
+	 * Override the default IndexedDB persistence implementation.
 	 */
-	onOperation?: (operation: Operation) => void;
+	persistence?: PersistenceImplementation;
+
 	/**
 	 * Enables experimental WeakRef usage to cull documents
 	 * from cache that aren't being used. This is a performance
@@ -121,7 +114,7 @@ export class ClientDescriptor<
 		// we can't initialize the storage
 		if (typeof window === 'undefined' && !init.indexedDb) {
 			throw new Error(
-				'A verdant client was initialized in an environment without IndexedDB. If you are using verdant in a server-rendered framework, you must enforce that all clients are initialized on the client-side, or you must provide some mock interface of IDBFactory to the ClientDescriptor options.',
+				'A Verdant client was initialized in an environment without IndexedDB. If you are using verdant in a server-rendered framework, you must enforce that all clients are initialized on the client-side, or you must provide some mock interface of IDBFactory to the ClientDescriptor options.',
 			);
 		}
 
@@ -130,17 +123,43 @@ export class ClientDescriptor<
 		}
 		this._initializing = true;
 		try {
-			let storage: ClientImpl;
-			if (init.schema.wip) {
-				storage = await this.initializeWIPDatabases(init);
-			} else {
-				storage = await this.initializeDatabases(init);
-				this.cleanupWIPDatabases(init);
-			}
-
-			this.resolveReady(storage);
-			this._resolvedValue = storage;
-			return storage;
+			const time = new Time(
+				new HybridLogicalClockTimestampProvider(),
+				init.schema.version,
+			);
+			let ctx: InitialContext = {
+				closing: false,
+				entityEvents: new EventSubscriber(),
+				globalEvents: new EventSubscriber(),
+				internalEvents: new EventSubscriber(),
+				log: init.log || noop,
+				migrations: init.migrations,
+				namespace: init.namespace,
+				originalNamespace: init.namespace,
+				schema: init.schema,
+				oldSchemas: init.oldSchemas,
+				time,
+				undoHistory: init.undoHistory || new UndoHistory(),
+				weakRef: (val) =>
+					init.EXPERIMENTAL_weakRefs
+						? new WeakRef(val)
+						: (new FakeWeakRef(val) as any),
+				patchCreator: new PatchCreator(() => time.now),
+				config: {
+					files: init.files,
+					sync: init.sync,
+					persistence: {
+						disableRebasing: init.disableRebasing,
+						rebaseTimeout: init.rebaseTimeout,
+					},
+				},
+				persistence: init.persistence || new IdbPersistence(init.indexedDb),
+			};
+			const context = await initializePersistence(ctx);
+			const client = new Client(context) as ClientImpl;
+			this.resolveReady(client);
+			this._resolvedValue = client;
+			return client;
 		} catch (err) {
 			if (err instanceof Error) {
 				this.rejectReady(err as Error);
@@ -153,156 +172,6 @@ export class ClientDescriptor<
 		}
 	};
 
-	private initializeDatabases = async (init: ClientDescriptorOptions) => {
-		const metadataVersion = init[METADATA_VERSION_KEY];
-		const { db: metaDb } = await openMetadataDatabase({
-			indexedDB: init.indexedDb,
-			log: init.log,
-			namespace: init.namespace,
-			metadataVersion,
-		});
-
-		const context: Omit<Context, 'documentDb' | 'getNow'> = {
-			namespace: this._namespace,
-			metaDb,
-			schema: init.schema,
-			log: init.log || (() => {}),
-			undoHistory: init.undoHistory || new UndoHistory(),
-			entityEvents: new EventSubscriber(),
-			globalEvents: new EventSubscriber(),
-			internalEvents: new EventSubscriber(),
-			weakRef: (value) => {
-				if (init.EXPERIMENTAL_weakRefs) {
-					return new WeakRef(value);
-				} else {
-					return new FakeWeakRef(value) as unknown as WeakRef<typeof value>;
-				}
-			},
-			migrations: init.migrations,
-			oldSchemas: init.oldSchemas,
-		};
-		const meta = new Metadata({
-			context,
-			disableRebasing: init.disableRebasing,
-			onOperation: init.onOperation,
-		});
-
-		// verify schema integrity
-		await meta.updateSchema(init.schema, init.overrideSchemaConflict);
-
-		const contextWithNow: Omit<Context, 'documentDb'> = Object.assign(context, {
-			getNow: () => meta.now,
-		});
-
-		const documentDb = await openQueryDatabase({
-			context: contextWithNow,
-			version: init.schema.version,
-			meta,
-			migrations: init.migrations,
-			indexedDB: init.indexedDb,
-		});
-
-		const fullContext: Context = Object.assign(contextWithNow, { documentDb });
-
-		const storage = new Client(
-			{
-				syncConfig: init.sync,
-				migrations: init.migrations,
-				files: init.files,
-			},
-			fullContext,
-			{
-				meta,
-			},
-		) as ClientImpl;
-
-		return storage;
-	};
-
-	private initializeWIPDatabases = async (init: ClientDescriptorOptions) => {
-		const schemaHash = hashObject(init.schema);
-		console.info(`WIP schema in use. Opening database with hash ${schemaHash}`);
-
-		const wipNamespace = `@@wip_${init.namespace}_${schemaHash}`;
-		const { db: metaDb } = await openWIPMetadataDatabase({
-			indexedDB: init.indexedDb,
-			log: init.log,
-			namespace: init.namespace,
-			wipNamespace: wipNamespace,
-		});
-
-		const context: Omit<Context, 'documentDb' | 'getNow'> = {
-			namespace: this._namespace,
-			metaDb,
-			schema: init.schema,
-			log: init.log || (() => {}),
-			undoHistory: init.undoHistory || new UndoHistory(),
-			entityEvents: new EventSubscriber(),
-			globalEvents: new EventSubscriber(),
-			internalEvents: new EventSubscriber(),
-			weakRef: (value) => {
-				if (init.EXPERIMENTAL_weakRefs) {
-					return new WeakRef(value);
-				} else {
-					return new FakeWeakRef(value) as unknown as WeakRef<typeof value>;
-				}
-			},
-			migrations: init.migrations,
-			oldSchemas: init.oldSchemas,
-		};
-		const meta = new Metadata({
-			context,
-			disableRebasing: init.disableRebasing,
-		});
-
-		const contextWithNow: Omit<Context, 'documentDb'> = Object.assign(context, {
-			getNow: () => meta.now,
-		});
-
-		// verify schema integrity
-		await meta.updateSchema(init.schema, init.overrideSchemaConflict);
-
-		const documentDb = await openWIPDatabase({
-			context: contextWithNow,
-			version: init.schema.version,
-			meta,
-			migrations: init.migrations,
-			indexedDB: init.indexedDb,
-			wipNamespace,
-		});
-
-		const fullContext: Context = Object.assign(contextWithNow, { documentDb });
-
-		const storage = new Client(
-			{
-				syncConfig: init.sync,
-				migrations: init.migrations,
-				files: init.files,
-			},
-			fullContext,
-			{
-				meta,
-			},
-		) as ClientImpl;
-
-		return storage;
-	};
-
-	private cleanupWIPDatabases = async (init: ClientDescriptorOptions) => {
-		const databaseInfo = await getAllDatabaseNamesAndVersions(init.indexedDb);
-		const wipDatabases = databaseInfo
-			.filter((db) => db.name?.startsWith('@@wip_'))
-			.map((db) => db.name!);
-		// don't clear a current WIP database.
-		const wipDatabasesToDelete = wipDatabases.filter(
-			(db) =>
-				!db.startsWith(`@@wip_${init.namespace}_${hashObject(init.schema)}`),
-		);
-		for (const db of wipDatabasesToDelete) {
-			await deleteDatabase(db, init.indexedDb);
-		}
-	};
-
 	get current() {
 		// exposing an immediate value if already resolved lets us
 		// skip the promise microtask when accessing this externally if

package/src/context/Time.ts

@@ -0,0 +1,35 @@
+import { TimestampProvider } from '@verdant-web/common';
+
+export class Time {
+	private overrideNow?: () => string;
+	constructor(
+		private base: TimestampProvider,
+		private version: number,
+	) {}
+
+	get now() {
+		return this.overrideNow ? this.overrideNow() : this.base.now(this.version);
+	}
+
+	withMigrationTime = async (version: number, run: () => Promise<void>) => {
+		this.overrideNow = () => {
+			return this.base.zero(version);
+		};
+		await run();
+		this.overrideNow = undefined;
+	};
+
+	update = this.base.update.bind(this.base);
+
+	nowWithVersion = (version: number) => {
+		return this.base.now(version);
+	};
+
+	get zero() {
+		return this.base.zero(this.version);
+	}
+
+	zeroWithVersion = (version: number) => {
+		return this.base.zero(version);
+	};
+}

package/src/context/context.ts

@@ -0,0 +1,200 @@
+import {
+	ClientMessage,
+	EventSubscriber,
+	FileData,
+	FileRef,
+	Migration,
+	ObjectIdentifier,
+	Operation,
+	PatchCreator,
+	StorageSchema,
+} from '@verdant-web/common';
+import { UndoHistory } from '../UndoHistory.js';
+import { Time } from './Time.js';
+import type { PersistenceQueries } from '../persistence/PersistenceQueries.js';
+import type { PersistenceMetadata } from '../persistence/PersistenceMetadata.js';
+import { PersistenceFiles } from '../persistence/PersistenceFiles.js';
+import {
+	PersistedFileData,
+	PersistenceImplementation,
+} from '../persistence/interfaces.js';
+
+/**
+ * Common components utilized across various client
+ * services.
+ */
+export interface Context {
+	namespace: string;
+	/**
+	 * when in WIP mode, namespace might be set to a temporary value. This will always point to the
+	 * namespace the user passed in.
+	 */
+	originalNamespace: string;
+	time: Time;
+
+	meta: PersistenceMetadata;
+	queries: PersistenceQueries;
+	files: PersistenceFiles;
+
+	undoHistory: UndoHistory;
+	schema: StorageSchema;
+	oldSchemas: StorageSchema[];
+	log: (
+		level: 'debug' | 'info' | 'warn' | 'error' | 'critical',
+		...args: any[]
+	) => void;
+	entityEvents: EventSubscriber<{
+		collectionsChanged: (names: string[]) => void;
+		documentChanged: (oid: ObjectIdentifier) => void;
+	}>;
+	internalEvents: EventSubscriber<{
+		/**
+		 * Fired when persisted data changes fundamentally, like resetting to 0,
+		 * or importing different data.
+		 */
+		persistenceReset: () => void;
+		filesDeleted: (files: FileRef[]) => void;
+		fileAdded: (file: FileData) => void;
+		[ev: `fileUploaded:${string}`]: () => void;
+	}>;
+	globalEvents: EventSubscriber<{
+		/**
+		 * A change from a future version of the application has been
+		 * witnessed. These changes are not applied but it indicates
+		 * the app has been updated and we should prompt the user
+		 * to reload or have their app user manually reload.
+		 *
+		 * The parameter is the timestamp of the future change.
+		 */
+		futureSeen: (timestamp: string) => void;
+		/**
+		 * The server requested this replica reset its state
+		 * completely. This can happen when the replica has
+		 * been offline for too long and reconnects.
+		 */
+		resetToServer: () => void;
+		/**
+		 * An operation has been processed by the system. This could be a locally sourced
+		 * operation or a remote operation from sync.
+		 */
+		operation: (operation: Operation) => void;
+		/**
+		 * Emitted when storage rebases history. This should never actually affect application behavior
+		 * or stored data, but is useful for debugging and testing.
+		 */
+		rebase: () => void;
+	}>;
+	weakRef<T extends object>(value: T): WeakRef<T>;
+	migrations: Migration<any>[];
+	closing: boolean;
+	patchCreator: PatchCreator;
+
+	config: {
+		files?: FileConfig;
+		sync?: SyncConfig;
+		persistence?: PersistenceConfig;
+	};
+
+	persistence: PersistenceImplementation;
+}
+
+export interface FileConfig {
+	/**
+	 * Override the heuristic for deciding when a deleted file can be cleaned up.
+	 * By default this waits 3 days since deletion, then deletes the file data.
+	 * If the file has been synchronized to a server, it could still be restored
+	 * if the server has not yet deleted it.
+	 */
+	canCleanupDeletedFile?: (file: PersistedFileData) => boolean;
+}
+
+export interface ServerSyncEndpointProviderConfig {
+	/**
+	 * The location of the endpoint used to retrieve an
+	 * authorization token for the client.
+	 */
+	authEndpoint?: string;
+	/**
+	 * A custom function to retrieve authorization
+	 * data. Use whatever fetching mechanism you want.
+	 */
+	fetchAuth?: () => Promise<{
+		accessToken: string;
+	}>;
+	/**
+	 * A spec-compliant fetch implementation. If not provided,
+	 * the global fetch will be used. authEndpoint will
+	 * be used to fetch the token.
+	 */
+	fetch?: typeof fetch;
+}
+
+export type SyncTransportMode = 'realtime' | 'pull';
+
+export interface SyncConfig<Profile = any, Presence = any>
+	extends ServerSyncEndpointProviderConfig {
+	/**
+	 * When a client first connects, it will use this presence value.
+	 */
+	initialPresence: Presence;
+	/**
+	 * Before connecting to the server, the local client will have
+	 * this value for their profile data. You can either cache and store
+	 * profile data from a previous connection or provide defaults like
+	 * empty strings.
+	 */
+	defaultProfile: Profile;
+
+	/**
+	 * Provide `false` to disable transport selection. Transport selection
+	 * automatically switches between HTTP and WebSocket based sync depending
+	 * on the number of peers connected. If a user is alone, they will use
+	 * HTTP push/pull to sync changes. If another user joins, both users will
+	 * be upgraded to websockets.
+	 *
+	 * Provide `peers-only` to only automatically use websockets if other
+	 * users connect, but not if another device for the current user connects.
+	 * By default, automatic transport selection will upgrade to websockets if
+	 * another device from the current user connects, but if realtime sync is
+	 * not necessary for such cases, you can save bandwidth by disabling this.
+	 *
+	 * Turning off this feature allows you more control over the transport
+	 * which can be useful for low-power devices or to save server traffic.
+	 * To modify transport modes manually, utilize `client.sync.setMode`.
+	 * The built-in behavior is essentially switching modes based on
+	 * the number of peers detected by client.sync.presence.
+	 */
+	automaticTransportSelection?: boolean | 'peers-only';
+	initialTransport?: SyncTransportMode;
+	autoStart?: boolean;
+	/**
+	 * Optionally specify an interval, in milliseconds, to poll the server
+	 * when in pull mode.
+	 */
+	pullInterval?: number;
+	/**
+	 * Presence updates are batched to reduce number of requests / messages
+	 * sent to the server. You can specify the batching time slice, in milliseconds,
+	 */
+	presenceUpdateBatchTimeout?: number;
+	/**
+	 * Experimental: sync messages over a broadcast channel between tabs.
+	 * Fixes tabs not reactively updating to changes when other tabs are open,
+	 * but is not yet thoroughly vetted.
+	 */
+	useBroadcastChannel?: boolean;
+	/**
+	 * Listen for outgoing messages from the client to the server.
+	 * Not sure why you want to do this, but be careful.
+	 */
+	onOutgoingMessage?: (message: ClientMessage) => void;
+
+	EXPERIMENTAL_backgroundSync?: boolean;
+}
+
+export interface PersistenceConfig {
+	disableRebasing?: boolean;
+	rebaseTimeout?: number;
+}
+
+export type InitialContext = Omit<Context, 'queries' | 'meta' | 'files'>;

package/src/entities/DocumentManager.ts

@@ -11,9 +11,6 @@ import {
 	AuthorizationKey,
 } from '@verdant-web/common';
 import { EntityCreateOptions, EntityStore } from '../entities/EntityStore.js';
-import { Metadata } from '../metadata/Metadata.js';
-import { Sync } from '../sync/Sync.js';
-import { Context } from '../context.js';
 import { ObjectEntity } from '../index.js';
 
 /**

package/src/entities/Entity.test.ts

@@ -1,16 +1,16 @@
 import { describe, expect, it, vi } from 'vitest';
 import { Entity } from './Entity.js';
 import { EntityFamilyMetadata } from './EntityMetadata.js';
-import { Context } from '../context.js';
+import { Context } from '../context/context.js';
 import { EntityStoreEvents } from './EntityStore.js';
 import { WeakEvent } from 'weak-event';
 import { FileManager } from '../files/FileManager.js';
 import {
 	NaiveTimestampProvider,
 	PatchCreator,
-	createRef,
 	groupPatchesByOid,
 } from '@verdant-web/common';
+import { Time } from '../context/Time.js';
 
 describe('Entity', () => {
 	const schema = {
@@ -60,18 +60,19 @@ describe('Entity', () => {
 			resetAll: new WeakEvent(),
 		};
 		const time = new NaiveTimestampProvider();
-		const mockContext = {
-			log: vi.fn(),
-			getNow: () => time.now(1),
-		} as any as Context;
 		const patchCreator = new PatchCreator(() => time.now(1));
+		const mockContext: Partial<Context> = {
+			log: vi.fn(),
+			time: new Time(time, 1),
+			patchCreator,
+		};
 		const entity = new Entity({
 			oid: 'test/1',
 			schema,
-			ctx: mockContext,
+			ctx: mockContext as Context,
 			storeEvents: events,
 			metadataFamily: new EntityFamilyMetadata({
-				ctx: mockContext,
+				ctx: mockContext as Context,
 				onPendingOperations,
 				rootOid: 'test/1',
 			}),
@@ -79,7 +80,6 @@ describe('Entity', () => {
 				add: vi.fn(),
 				get: vi.fn(),
 			} as any as FileManager,
-			patchCreator,
 			readonlyKeys: ['id'],
 			deleteSelf: vi.fn(),
 		});

package/src/entities/Entity.ts

@@ -1,12 +1,9 @@
 import {
-	DocumentBaseline,
 	EntityValidationProblem,
 	EventSubscriber,
 	ObjectIdentifier,
 	Operation,
-	PatchCreator,
 	StorageFieldSchema,
-	StorageFieldsSchema,
 	assert,
 	assignOid,
 	cloneDeep,
@@ -20,15 +17,13 @@ import {
 	isFile,
 	isNullable,
 	isObject,
-	isObjectRef,
-	isPrunePoint,
 	isRef,
 	maybeGetOid,
 	memoByKeys,
 	traverseCollectionFieldsAndApplyDefaults,
 	validateEntityField,
 } from '@verdant-web/common';
-import { Context } from '../context.js';
+import { Context } from '../context/context.js';
 import { FileManager } from '../files/FileManager.js';
 import { processValueFiles } from '../files/utils.js';
 import { EntityFile } from '../index.js';
@@ -37,7 +32,6 @@ import { EntityFamilyMetadata, EntityMetadataView } from './EntityMetadata.js';
 import {
 	BaseEntityValue,
 	DataFromInit,
-	DeepPartial,
 	EntityChange,
 	EntityEvents,
 	ListEntity,
@@ -58,7 +52,6 @@ export interface EntityInit {
 	files: FileManager;
 	readonlyKeys?: string[];
 	fieldPath?: (string | number)[];
-	patchCreator: PatchCreator;
 	storeEvents: EntityStoreEvents;
 	deleteSelf: () => void;
 }
@@ -84,7 +77,6 @@ export class Entity<
 	private parent: Entity | undefined;
 	private ctx;
 	private files;
-	private patchCreator;
 	private storeEvents;
 
 	// an internal representation of this Entity.
@@ -110,7 +102,6 @@ export class Entity<
 		metadataFamily,
 		readonlyKeys,
 		files,
-		patchCreator,
 		storeEvents,
 		deleteSelf,
 	}: EntityInit) {
@@ -128,7 +119,6 @@ export class Entity<
 			new EntityCache({
 				initial: [this],
 			});
-		this.patchCreator = patchCreator;
 		this.metadataFamily = metadataFamily;
 		this.storeEvents = storeEvents;
 		this.parent = parent;
@@ -160,6 +150,10 @@ export class Entity<
 		return this.metadataFamily.get(this.oid);
 	}
 
+	private get patchCreator() {
+		return this.ctx.patchCreator;
+	}
+
 	/**
 	 * The view of this Entity, not including nested
 	 * entities (that's the snapshot - see #getSnapshot())
@@ -585,7 +579,6 @@ export class Entity<
 			ctx: this.ctx,
 			files: this.files,
 			fieldPath: [...this.fieldPath, key],
-			patchCreator: this.patchCreator,
 			storeEvents: this.storeEvents,
 			deleteSelf: this.delete.bind(this, key),
 		});
@@ -635,6 +628,7 @@ export class Entity<
 				}
 				const file = this.files.get(child.id, {
 					downloadRemote: !!fieldSchema.downloadRemote,
+					ctx: this.ctx,
 				});
 
 				// FIXME: this seems bad and inconsistent

package/src/entities/EntityCache.ts

@@ -26,15 +26,6 @@ export class EntityCache {
 		return this.cache.has(oid);
 	};
 
-	getFile = (id: string, options: { downloadRemote: boolean }): EntityFile => {
-		if (this.cache.has(id)) {
-			return this.cache.get(id)! as EntityFile;
-		}
-		const file = new EntityFile(id, options);
-		this.cache.set(id, file);
-		return file;
-	};
-
 	getCached = (oid: string) => {
 		return this.cache.get(oid);
 	};