@verdant-web/store 3.12.0 → 4.0.0-next.0

package/src/persistence/persistence.ts

@@ -0,0 +1,223 @@
+import { EventSubscriber, getOidRoot, VerdantError } from '@verdant-web/common';
+import { Context, InitialContext } from '../context/context.js';
+import { getWipNamespace } from '../utils/wip.js';
+import { ExportedData } from './interfaces.js';
+import { PersistenceFiles } from './PersistenceFiles.js';
+import { PersistenceMetadata } from './PersistenceMetadata.js';
+import { PersistenceQueries } from './PersistenceQueries.js';
+
+export async function initializePersistence(
+	ctx: InitialContext,
+): Promise<Context> {
+	let context = ctx as any as Context;
+	if (ctx.schema.wip) {
+		// this is a WIP database, so we need to create a new namespace for the WIP data.
+		context.namespace = getWipNamespace(ctx.originalNamespace, ctx.schema);
+		context.log('info', 'Switched to WIP namespace', context.namespace);
+		// check if this WIP database is already in use
+		const namespaces = await context.persistence.getNamespaces();
+
+		if (!namespaces.includes(context.namespace)) {
+			// copy all data to WIP namespace -- from the current version of local
+			// data, not the WIP schema version. this may not be n-1, we might
+			// be loading a WIP schema over older data.
+			const currentVersion = await context.persistence.getNamespaceVersion(
+				context.originalNamespace,
+			);
+
+			if (currentVersion === 0) {
+				// there is no existing data. nothing to copy.
+			} else {
+				const currentSchema = ctx.oldSchemas?.find(
+					(s) => s.version === currentVersion,
+				);
+				if (!currentSchema) {
+					throw new VerdantError(
+						VerdantError.Code.MigrationPathNotFound,
+						undefined,
+						`Trying to open WIP database for version ${ctx.schema.version}, but the current local data is version ${currentVersion} and a historical schema for that version is not available.`,
+					);
+				}
+				context.log(
+					'info',
+					`Copying data from ${context.originalNamespace} to ${context.namespace}`,
+				);
+				await context.persistence.copyNamespace(
+					context.originalNamespace,
+					context.namespace,
+					{
+						...context,
+						schema: currentSchema,
+					},
+				);
+			}
+		}
+	}
+
+	context.log('info', 'Opening persistence metadata');
+	context.meta = new PersistenceMetadata(
+		await ctx.persistence.openMetadata(ctx),
+		ctx,
+	);
+
+	context.log('info', 'Opening persistence files');
+	context.files = new PersistenceFiles(
+		await ctx.persistence.openFiles(context),
+		context,
+	);
+
+	context.log('info', 'Opening persistence queries');
+	context.queries = new PersistenceQueries(
+		await ctx.persistence.openQueries(context),
+		context,
+	);
+
+	if (!ctx.schema.wip) {
+		const namespaces = await ctx.persistence.getNamespaces();
+		// cleanup old WIP databases
+		for (const namespace of namespaces) {
+			if (namespace.startsWith(`@@wip_`)) {
+				ctx.log('debug', 'Cleaning up old WIP namespace', namespace);
+				await ctx.persistence.deleteNamespace(namespace, ctx);
+			}
+		}
+	}
+
+	return context;
+}
+
+export async function importPersistence(
+	ctx: Context,
+	exportedData: ExportedData,
+): Promise<void> {
+	ctx.log('info', 'Importing data from export');
+	// open persistence at the version of the import
+	const exportedSchema = ctx.oldSchemas?.find(
+		(s) => s.version === exportedData.data.schemaVersion,
+	);
+	if (!exportedSchema) {
+		// this means the user hasn't upgraded their CLI/client to include old schemas, or
+		// this exported data comes from a version which has since been shortcut.
+		throw new Error(
+			`Could not find schema for version ${exportedData.data.schemaVersion}`,
+		);
+	}
+
+	// using a new namespace to put all of this into a temporary zone
+	const importedNamespace = `@@import_${Date.now()}`;
+
+	const importedContext = await initializePersistence({
+		...ctx,
+		schema: exportedSchema,
+		namespace: importedNamespace,
+		originalNamespace: importedNamespace,
+		// no-op entity events -- don't need to inform queries of changes.
+		entityEvents: new EventSubscriber(),
+		config: {
+			...ctx.config,
+			persistence: {
+				...ctx.config.persistence,
+				disableRebasing: true,
+			},
+		},
+	});
+	// load imported data into persistence
+	await importedContext.meta.resetFrom(exportedData.data);
+	// need to write indexes here!
+	const affectedOids = new Set<string>();
+	for (const baseline of exportedData.data.baselines) {
+		affectedOids.add(getOidRoot(baseline.oid));
+	}
+	for (const operation of exportedData.data.operations) {
+		affectedOids.add(getOidRoot(operation.oid));
+	}
+	const toSave = await Promise.all(
+		Array.from(affectedOids).map(async (oid) => {
+			const snapshot = await importedContext.meta.getDocumentSnapshot(oid);
+			return {
+				oid,
+				getSnapshot: () => snapshot,
+			};
+		}),
+	);
+	await importedContext.queries.saveEntities(toSave);
+	await importedContext.files.import(exportedData);
+
+	ctx.log('debug', 'Imported data into temporary namespace', importedNamespace);
+
+	// shut down the imported databases
+	await importedContext.queries.dispose();
+	await importedContext.meta.dispose();
+	await importedContext.files.dispose();
+
+	if (exportedSchema.version !== ctx.schema.version) {
+		// an upgrade of the imported data is needed ; it's an older version
+		// of the schema.
+		ctx.log('debug', 'Shut down imported databases');
+
+		// upgrade the imported data to the latest schema
+		const currentSchema = ctx.schema;
+		const upgradedContext = await initializePersistence({
+			...importedContext,
+			schema: currentSchema,
+		});
+
+		ctx.log('debug', 'Upgraded imported data to current schema');
+
+		await upgradedContext.queries.dispose();
+		await upgradedContext.meta.dispose();
+		await upgradedContext.files.dispose();
+
+		ctx.log('debug', 'Shut down upgraded databases');
+	}
+
+	// copy the imported data into the current namespace
+	await ctx.persistence.copyNamespace(importedNamespace, ctx.namespace, ctx);
+
+	// verify integrity -- this can only be done if imported data was same
+	// version as current schema, because migrations could add or remove
+	// operations. still, it's a good sanity check.
+	if (exportedData.data.schemaVersion === ctx.schema.version) {
+		const stats = await ctx.meta.stats();
+		if (stats.operationsSize.count !== exportedData.data.operations.length) {
+			ctx.log(
+				'critical',
+				'Imported operations count mismatch',
+				'expected',
+				exportedData.data.operations.length,
+				'actual',
+				stats.operationsSize.count,
+			);
+			throw new VerdantError(
+				VerdantError.Code.ImportFailed,
+				undefined,
+				'Imported operations count mismatch',
+			);
+		}
+		if (stats.baselinesSize.count !== exportedData.data.baselines.length) {
+			ctx.log(
+				'critical',
+				'Imported documents count mismatch',
+				'expected',
+				exportedData.data.baselines.length,
+				'actual',
+				stats.baselinesSize.count,
+			);
+			throw new VerdantError(
+				VerdantError.Code.ImportFailed,
+				undefined,
+				'Imported documents count mismatch',
+			);
+		}
+	}
+
+	ctx.log('debug', 'Data copied to primary namespace');
+
+	// cleanup the imported namespace
+	await ctx.persistence.deleteNamespace(importedNamespace, ctx);
+
+	ctx.log('debug', 'Deleted temporary namespace');
+
+	ctx.internalEvents.emit('persistenceReset');
+	ctx.log('info', 'Data imported successfully');
+}

package/src/queries/BaseQuery.ts

@@ -1,5 +1,5 @@
 import { EventSubscriber } from '@verdant-web/common';
-import { Context } from '../context.js';
+import { Context } from '../context/context.js';
 import { Entity } from '../entities/Entity.js';
 import { Disposable } from '../utils/Disposable.js';
 import { filterResultSet } from './utils.js';
@@ -253,4 +253,8 @@ export abstract class BaseQuery<T> extends Disposable {
 	[ON_ALL_UNSUBSCRIBED] = (handler: (query: BaseQuery<T>) => void) => {
 		this._allUnsubscribedHandler = handler;
 	};
+
+	get __rawValue() {
+		return this._rawValue;
+	}
 }

package/src/queries/CollectionQueries.ts

@@ -3,7 +3,7 @@ import {
 	CollectionFilter,
 	hashObject,
 } from '@verdant-web/common';
-import { Context } from '../context.js';
+import { Context } from '../context/context.js';
 import { EntityStore } from '../entities/EntityStore.js';
 import { GetQuery } from './GetQuery.js';
 import { QueryCache } from './QueryCache.js';
@@ -12,7 +12,7 @@ import { FindPageQuery } from './FindPageQuery.js';
 import { FindInfiniteQuery } from './FindInfiniteQuery.js';
 import { FindAllQuery } from './FindAllQuery.js';
 import { DocumentManager } from '../entities/DocumentManager.js';
-import { Entity, ObjectEntity } from '../index.js';
+import { ObjectEntity } from '../index.js';
 import { UPDATE } from './BaseQuery.js';
 
 export class CollectionQueries<

package/src/queries/FindAllQuery.ts

@@ -1,6 +1,5 @@
 import { CollectionFilter } from '@verdant-web/common';
 import { BaseQuery, BaseQueryOptions, UPDATE } from './BaseQuery.js';
-import { findAllOids } from './dbQueries.js';
 import { areIndexesEqual } from './utils.js';
 
 export class FindAllQuery<T> extends BaseQuery<T[]> {
@@ -24,10 +23,9 @@ export class FindAllQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const oids = await findAllOids({
+		const { result: oids } = await this.context.queries.findAllOids({
 			collection: this.collection,
 			index: this.index,
-			context: this.context,
 		});
 		this.context.log(
 			'debug',

package/src/queries/FindInfiniteQuery.ts

@@ -1,6 +1,5 @@
 import { CollectionFilter } from '@verdant-web/common';
 import { BaseQuery, BaseQueryOptions, UPDATE } from './BaseQuery.js';
-import { findPageOfOids } from './dbQueries.js';
 import { areIndexesEqual } from './utils.js';
 
 export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
@@ -38,9 +37,8 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await findPageOfOids({
+		const { result, hasNextPage } = await this.context.queries.findAllOids({
 			collection: this.collection,
-			context: this.context,
 			limit: this._pageSize * this._upToPage,
 			offset: 0,
 			index: this.index,
@@ -50,9 +48,8 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	};
 
 	public loadMore = async () => {
-		const { result, hasNextPage } = await findPageOfOids({
+		const { result, hasNextPage } = await this.context.queries.findAllOids({
 			collection: this.collection,
-			context: this.context,
 			limit: this._pageSize,
 			offset: this._pageSize * this._upToPage,
 			index: this.index,

package/src/queries/FindOneQuery.ts

@@ -1,6 +1,5 @@
 import { CollectionFilter } from '@verdant-web/common';
 import { BaseQuery, BaseQueryOptions, UPDATE } from './BaseQuery.js';
-import { findOneOid } from './dbQueries.js';
 import { areIndexesEqual } from './utils.js';
 
 export class FindOneQuery<T> extends BaseQuery<T | null> {
@@ -24,10 +23,9 @@ export class FindOneQuery<T> extends BaseQuery<T | null> {
 	}
 
 	protected run = async () => {
-		const oid = await findOneOid({
+		const oid = await this.context.queries.findOneOid({
 			collection: this.collection,
 			index: this.index,
-			context: this.context,
 		});
 		this.setValue(oid ? await this.hydrate(oid) : null);
 	};

package/src/queries/FindPageQuery.ts

@@ -1,6 +1,5 @@
 import { CollectionFilter } from '@verdant-web/common';
 import { BaseQuery, BaseQueryOptions, UPDATE } from './BaseQuery.js';
-import { findPageOfOids } from './dbQueries.js';
 import { areIndexesEqual } from './utils.js';
 
 export class FindPageQuery<T> extends BaseQuery<T[]> {
@@ -49,10 +48,9 @@ export class FindPageQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await findPageOfOids({
+		const { result, hasNextPage } = await this.context.queries.findAllOids({
 			collection: this.collection,
 			index: this.index,
-			context: this.context,
 			limit: this._pageSize,
 			offset: this._page * this._pageSize,
 		});

package/src/queries/QueryCache.ts

@@ -1,4 +1,4 @@
-import { Context } from '../context.js';
+import { Context } from '../context/context.js';
 import { Disposable } from '../utils/Disposable.js';
 import { BaseQuery, ON_ALL_UNSUBSCRIBED } from './BaseQuery.js';
 
@@ -18,6 +18,12 @@ export class QueryCache extends Disposable {
 
 		this._evictionTime = evictionTime;
 		this.context = context;
+		this.addDispose(
+			this.context.internalEvents.subscribe(
+				'persistenceReset',
+				this.forceRefreshAll,
+			),
+		);
 	}
 
 	get<T extends BaseQuery<any>>(key: string): T | null {
@@ -40,6 +46,7 @@ export class QueryCache extends Disposable {
 			update?.(existing);
 			return existing;
 		}
+		this.context.log('debug', 'QueryCache: creating new query', key);
 		return this.set(create());
 	}
 
@@ -55,11 +62,23 @@ export class QueryCache extends Disposable {
 	};
 
 	dropAll = () => {
+		this.context.log(
+			'debug',
+			'QueryCache: drop all',
+			this._cache.size,
+			'queries',
+		);
 		this._cache.forEach((query) => query.dispose());
 		this._cache.clear();
 	};
 
 	forceRefreshAll = () => {
+		this.context.log(
+			'debug',
+			'QueryCache: force refresh all',
+			this._cache.size,
+			'queries',
+		);
 		this._cache.forEach((q) => q.execute());
 	};
 }

package/src/sync/FileSync.ts

@@ -1,9 +1,10 @@
 import { FileData } from '@verdant-web/common';
 import { ServerSyncEndpointProvider } from './ServerSyncEndpointProvider.js';
+import { Context } from '../context/context.js';
+import { Disposable } from '../utils/Disposable.js';
 
 export interface FileUploadResult {
 	success: boolean;
-	retry: boolean;
 	error?: string;
 }
 
@@ -14,26 +15,47 @@ export type FilePullResult =
 	  }
 	| {
 			success: false;
-			retry: boolean;
 			error?: any;
 	  };
 
-export class FileSync {
+export class FileSync extends Disposable {
 	private endpointProvider: ServerSyncEndpointProvider;
-	private log: (...args: any[]) => any;
+	private ctx;
 
 	constructor({
 		endpointProvider,
-		log,
+		ctx,
 	}: {
 		endpointProvider: ServerSyncEndpointProvider;
-		log: (...args: any[]) => any;
+		ctx: Context;
 	}) {
+		super();
 		this.endpointProvider = endpointProvider;
-		this.log = log;
+		this.ctx = ctx;
+		this.addDispose(
+			ctx.internalEvents.subscribe('fileAdded', this.onFileAdded),
+		);
 	}
 
-	uploadFile = async (data: FileData): Promise<FileUploadResult> => {
+	private onFileAdded = async (data: FileData) => {
+		if (data.remote) return;
+		this.ctx.log('debug', 'Uploading file', data.id, data.name);
+		try {
+			await this.uploadFile(data);
+			this.ctx.internalEvents.emit(`fileUploaded:${data.id}`);
+		} catch (e) {
+			this.ctx.log('error', 'File upload failed', e);
+		}
+	};
+
+	/**
+	 * Attempts to upload a file to the sync server. Will be retried
+	 * according to retry config.
+	 */
+	uploadFile = async (
+		data: FileData,
+		retries: { max: number; current: number } = { current: 0, max: 3 },
+	): Promise<FileUploadResult> => {
 		const file = data.file;
 
 		if (!file) {
@@ -58,31 +80,54 @@ export class FileSync {
 			});
 
 			if (response.ok) {
-				this.log('info', 'File upload successful');
+				this.ctx.log('info', 'File upload successful');
 				return {
 					success: true,
-					retry: false,
 				};
 			} else {
 				const responseText = await response.text();
-				this.log('error', 'File upload failed', response.status, responseText);
+				this.ctx.log(
+					'error',
+					'File upload failed',
+					response.status,
+					responseText,
+				);
+				if (response.status < 500 || retries.current >= retries.max) {
+					return {
+						success: false,
+						error: `Failed to upload file: ${response.status} ${responseText}`,
+					};
+				}
+				await new Promise((resolve) => setTimeout(resolve, 1000));
+				return this.uploadFile(data, {
+					max: retries.max,
+					current: retries.current + 1,
+				});
+			}
+		} catch (e) {
+			this.ctx.log('error', 'File upload failed', e);
+			if (retries.current >= retries.max) {
 				return {
 					success: false,
-					retry: response.status >= 500,
-					error: `Failed to upload file: ${response.status} ${responseText}`,
+					error: (e as Error).message,
 				};
 			}
-		} catch (e) {
-			this.log('error', 'File upload failed', e);
-			return {
-				success: false,
-				retry: true,
-				error: (e as Error).message,
-			};
+			await new Promise((resolve) => setTimeout(resolve, 1000));
+			return this.uploadFile(data, {
+				max: retries.max,
+				current: retries.current + 1,
+			});
 		}
 	};
 
-	getFile = async (id: string): Promise<FilePullResult> => {
+	/**
+	 * Pulls a file from the server by its ID. Will be retried
+	 * according to retry config.
+	 */
+	getFile = async (
+		id: string,
+		retries: { current: number; max: number } = { current: 0, max: 3 },
+	): Promise<FilePullResult> => {
 		const { files: fileEndpoint, token } =
 			await this.endpointProvider.getEndpoints();
 
@@ -103,24 +148,42 @@ export class FileSync {
 					data,
 				};
 			} else {
-				this.log(
+				this.ctx.log(
 					'error',
 					'File information fetch failed',
 					response.status,
 					await response.text(),
 				);
+				if (
+					(response.status < 500 && response.status !== 404) ||
+					retries.current >= retries.max
+				) {
+					return {
+						success: false,
+						error: `Failed to fetch file: ${response.status}`,
+					};
+				}
+
+				await new Promise((resolve) => setTimeout(resolve, 1000));
+				return this.getFile(id, {
+					current: retries.current + 1,
+					max: retries.max,
+				});
+			}
+		} catch (e) {
+			this.ctx.log('error', 'File information fetch failed', e);
+			if (retries.current >= retries.max) {
 				return {
 					success: false,
-					retry: response.status >= 500 || response.status === 404,
+					error: (e as Error).message,
 				};
 			}
-		} catch (e) {
-			this.log('error', 'File information fetch failed', e);
-			return {
-				success: false,
-				error: e,
-				retry: true,
-			};
+
+			await new Promise((resolve) => setTimeout(resolve, 1000));
+			return this.getFile(id, {
+				current: retries.current + 1,
+				max: retries.max,
+			});
 		}
 	};
 }

package/src/sync/PresenceManager.ts

@@ -7,10 +7,8 @@ import {
 	initialInternalPresence,
 } from '@verdant-web/common';
 import type { UserInfo } from '../index.js';
-import {
-	LocalReplicaInfo,
-	LocalReplicaStore,
-} from '../metadata/LocalReplicaStore.js';
+import { Context } from '../context/context.js';
+import { LocalReplicaInfo } from '../persistence/interfaces.js';
 
 export const HANDLE_MESSAGE = Symbol('handleMessage');
 
@@ -77,12 +75,12 @@ export class PresenceManager<
 		initialPresence,
 		updateBatchTimeout = 200,
 		defaultProfile,
-		replicaStore,
+		ctx,
 	}: {
 		initialPresence: Presence;
 		defaultProfile: Profile;
 		updateBatchTimeout?: number;
-		replicaStore: LocalReplicaStore;
+		ctx: Context;
 	}) {
 		super();
 		this.self.presence = initialPresence;
@@ -92,7 +90,7 @@ export class PresenceManager<
 		this.self.replicaId = '';
 
 		// set the local replica ID as soon as it's loaded
-		replicaStore.get().then((info) => {
+		ctx.meta.getLocalReplica().then((info) => {
 			this.self.replicaId = info.id;
 		});