@verdant-web/store 3.2.1 → 3.3.0

package/src/client/Client.ts

@@ -21,6 +21,7 @@ import { EntityStore } from '../entities/EntityStore.js';
 import { NoSync, ServerSync, ServerSyncOptions, Sync } from '../sync/Sync.js';
 import { CollectionQueries } from '../queries/CollectionQueries.js';
 import { QueryCache } from '../queries/QueryCache.js';
+import { ReturnedFileData } from '../files/FileStorage.js';
 
 interface ClientConfig<Presence = any> {
 	syncConfig?: ServerSyncOptions<Presence>;
@@ -182,13 +183,6 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		return this.context.undoHistory;
 	}
 
-	/**
-	 * @deprecated - use client.sync.presence instead
-	 */
-	get presence() {
-		return this.sync.presence;
-	}
-
 	/**
 	 * Batch multiple operations together to be executed in a single transaction.
 	 * The changes made will not be included in the same undo history step as
@@ -276,21 +270,88 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		await deleteAllDatabases(this.namespace, indexedDB);
 	};
 
-	export = async () => {
+	export = async (
+		{ downloadRemoteFiles }: { downloadRemoteFiles?: boolean } = {
+			downloadRemoteFiles: true,
+		},
+	) => {
+		this.context.log('info', 'Exporting data...');
 		const metaExport = await this.meta.export();
-		return Buffer.from(JSON.stringify(metaExport));
+		const filesExport = await this._fileManager.exportAll(downloadRemoteFiles);
+		// split files into data and files
+		const fileData: Array<Omit<ReturnedFileData, 'file'>> = [];
+		const files: Array<File> = [];
+
+		for (const fileExport of filesExport) {
+			const file = fileExport.file;
+			delete fileExport.file;
+			fileData.push(fileExport);
+			if (file) {
+				// rename with ID
+				const asFile = new File(
+					[file],
+					this.getFileExportName(fileExport.name, fileExport.id),
+					{
+						type: fileExport.type,
+					},
+				);
+				files.push(asFile);
+			} else {
+				this.context.log(
+					'warn',
+					`File ${fileExport.id} was could not be loaded locally or from the server. It will be missing in the export.`,
+				);
+			}
+		}
+		return {
+			data: metaExport,
+			fileData,
+			files,
+		};
 	};
 
-	import = async (buffer: Buffer) => {
-		this.context.log('Importing data...');
+	private getFileExportName = (originalFileName: string, id: string) => {
+		return `${id}___${originalFileName}`;
+	};
+
+	private parseFileExportname = (name: string) => {
+		const [id, originalFileName] = name.split('___');
+		return { id, originalFileName };
+	};
+
+	import = async ({
+		data,
+		fileData,
+		files,
+	}: {
+		data: ExportData;
+		fileData: Array<Omit<ReturnedFileData, 'file'>>;
+		files: File[];
+	}) => {
+		this.context.log('info', 'Importing data...');
 		// close the document DB
 		await closeDatabase(this.context.documentDb);
 
-		const metaExport = JSON.parse(buffer.toString()) as ExportData;
-		await this.meta.resetFrom(metaExport);
+		await this.meta.resetFrom(data);
+		// re-attach files to their file data and import
+		const fileToIdMap = new Map(
+			files.map((file) => {
+				const { id } = this.parseFileExportname(file.name);
+				return [id, file];
+			}),
+		);
+		const importedFiles: ReturnedFileData[] = fileData.map((fileData) => {
+			const file = fileToIdMap.get(fileData.id);
+
+			return {
+				...fileData,
+				file,
+			};
+		});
+		await this._fileManager.importAll(importedFiles);
 		// now delete the document DB, open it to the specified version
 		// and run migrations to get it to the latest version
-		const version = metaExport.schema.version;
+		const version = data.schema.version;
 		const deleteReq = indexedDB.deleteDatabase(
 			[this.namespace, 'collections'].join('_'),
 		);
@@ -300,7 +361,18 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		});
 		// reset our context to the imported schema for now
 		const currentSchema = this.context.schema;
-		this.context.schema = metaExport.schema;
+		if (currentSchema.version !== version) {
+			// TODO: support importing older schema data - this will
+			// require being able to migrate that data, which requires
+			// a "live" schema for that version. the client does not currently
+			// receive historical schemas, although they should be available
+			// if the CLI was used.
+			// importing from older versions is also tricky because
+			// migration shortcuts mean that versions could get marooned.
+			throw new Error(
+				`Only exports from the current schema version can be imported`,
+			);
+		}
 		// now open the document DB empty at the specified version
 		// and initialize it from the meta DB
 		this.context.documentDb = await openDocumentDatabase({
@@ -313,8 +385,8 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		// re-initialize data
 		this.context.log('Re-initializing data from imported data...');
 		await this._entities.addData({
-			operations: metaExport.operations,
-			baselines: metaExport.baselines,
+			operations: data.operations,
+			baselines: data.baselines,
 			reset: true,
 		});
 		// close the database and reopen to latest version, applying
@@ -331,4 +403,20 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		});
 		this.context.internalEvents.emit('documentDbChanged', this.documentDb);
 	};
+
+	/**
+	 * Export all data, then re-import it. This might resolve
+	 * some issues with the local database, but it should
+	 * only be done as a second-to-last resort. The last resort
+	 * would be __dangerous__resetLocal on ClientDescriptor, which
+	 * clears all local data.
+	 *
+	 * Unlike __dangerous__resetLocal, this method allows local-only
+	 * clients to recover data, whereas __dangerous__resetLocal only
+	 * lets networked clients recover from the server.
+	 */
+	__dangerous__hardReset = async () => {
+		const exportData = await this.export();
+		await this.import(exportData);
+	};
 }

package/src/files/FileManager.ts

@@ -3,7 +3,11 @@ import { Context } from '../context.js';
 import { Metadata } from '../metadata/Metadata.js';
 import { Sync } from '../sync/Sync.js';
 import { EntityFile, MARK_FAILED, UPDATE } from './EntityFile.js';
-import { FileStorage, ReturnedFileData } from './FileStorage.js';
+import {
+	FileStorage,
+	ReturnedFileData,
+	StoredFileData,
+} from './FileStorage.js';
 
 /**
  * Default: if file was deleted > 3 days ago
@@ -153,6 +157,36 @@ export class FileManager {
 		return this.storage.listUnsynced();
 	};
 
+	exportAll = async (downloadRemote = false) => {
+		const storedFiles = await this.storage.getAll();
+		if (downloadRemote) {
+			for (const storedFile of storedFiles) {
+				// if it doesn't have a buffer, we need to read
+				// one from the server
+				if (!storedFile.file && storedFile.url) {
+					try {
+						const blob = await fetch(storedFile.url, {
+							method: 'GET',
+							credentials: 'include',
+						}).then((r) => r.blob());
+						storedFile.file = blob;
+					} catch (err) {
+						this.context.log(
+							'error',
+							"Failed to download file to cache it locally. The file will still be available using its URL. Check the file server's CORS configuration.",
+							err,
+						);
+					}
+				}
+			}
+		}
+		return storedFiles;
+	};
+
+	importAll = async (files: ReturnedFileData[]) => {
+		await Promise.all(files.map((file) => this.add(file)));
+	};
+
 	private onOnlineChange = async (online: boolean) => {
 		// if online, try to upload any unsynced files
 		if (online) {

package/src/files/FileStorage.ts

@@ -1,13 +1,14 @@
 import { FileData } from '@verdant-web/common';
 import { IDBService } from '../IDBService.js';
 import { fileToArrayBuffer } from './utils.js';
+import { getAllFromObjectStores } from '../idb.js';
 
 /**
  * When stored in IDB, replace the file blob with an array buffer
  * since it's more compatible, and replace remote boolean with
  * a string since IDB doesn't support boolean indexes.
  */
-interface StoredFileData extends Omit<FileData, 'remote' | 'file'> {
+export interface StoredFileData extends Omit<FileData, 'remote' | 'file'> {
 	remote: 'true' | 'false';
 	buffer?: ArrayBuffer;
 	deletedAt: number | null;
@@ -180,6 +181,11 @@ export class FileStorage extends IDBService {
 			{ mode: 'readwrite', transaction },
 		);
 	};
+
+	getAll = async () => {
+		const [files] = await getAllFromObjectStores(this.db, ['files']);
+		return files.map(this.hydrateFileData);
+	};
 }
 
 export function arrayBufferToBlob(buffer: ArrayBuffer, type: string) {

package/src/sync/PushPullSync.ts

@@ -19,6 +19,7 @@ export class PushPullSync
 	readonly presence: PresenceManager;
 	private endpointProvider;
 	private heartbeat;
+	private fetch;
 
 	readonly mode = 'pull';
 	private log;
@@ -33,18 +34,21 @@ export class PushPullSync
 		presence,
 		interval = 15 * 1000,
 		log = () => {},
+		fetch = window.fetch,
 	}: {
 		endpointProvider: ServerSyncEndpointProvider;
 		meta: Metadata;
 		presence: PresenceManager;
 		interval?: number;
 		log?: (...args: any[]) => any;
+		fetch?: typeof window.fetch;
 	}) {
 		super();
 		this.log = log;
 		this.meta = meta;
 		this.presence = presence;
 		this.endpointProvider = endpointProvider;
+		this.fetch = fetch;
 
 		this.heartbeat = new Heartbeat({
 			interval,
@@ -65,7 +69,7 @@ export class PushPullSync
 		this.log('Sending sync request', messages);
 		try {
 			const { http: host, token } = await this.endpointProvider.getEndpoints();
-			const response = await fetch(host, {
+			const response = await this.fetch(host, {
 				method: 'POST',
 				headers: {
 					'Content-Type': 'application/json',

package/src/sync/ServerSyncEndpointProvider.ts

@@ -8,12 +8,18 @@ export interface ServerSyncEndpointProviderConfig {
 	 */
 	authEndpoint?: string;
 	/**
-	 * A custom fetch function to retrieve authorization
-	 * data.
+	 * A custom function to retrieve authorization
+	 * data. Use whatever fetching mechanism you want.
 	 */
 	fetchAuth?: () => Promise<{
 		accessToken: string;
 	}>;
+	/**
+	 * A spec-compliant fetch implementation. If not provided,
+	 * the global fetch will be used. authEndpoint will
+	 * be used to fetch the token.
+	 */
+	fetch?: typeof fetch;
 }
 
 export class ServerSyncEndpointProvider {
@@ -42,7 +48,8 @@ export class ServerSyncEndpointProvider {
 		if (this.config.fetchAuth) {
 			result = await this.config.fetchAuth();
 		} else {
-			result = await fetch(this.config.authEndpoint!, {
+			const fetchImpl = this.config.fetch || fetch;
+			result = await fetchImpl(this.config.authEndpoint!, {
 				credentials: 'include',
 			}).then((res) => {
 				if (!res.ok) {

package/src/sync/Sync.ts

@@ -26,12 +26,7 @@ export type SyncTransportEvents = SyncEvents & {
 	message: (message: ServerMessage) => void;
 };
 
-export interface SyncTransport {
-	subscribe(
-		event: 'onlineChange',
-		handler: (online: boolean) => void,
-	): () => void;
-
+export interface SyncTransport extends EventSubscriber<SyncTransportEvents> {
 	readonly presence: PresenceManager;
 
 	readonly mode: SyncTransportMode;
@@ -50,13 +45,23 @@ export interface SyncTransport {
 	readonly status: 'active' | 'paused';
 }
 
-export interface Sync<Presence = any, Profile = any> extends SyncTransport {
+export interface Sync<Presence = any, Profile = any>
+	extends EventSubscriber<SyncEvents> {
 	setMode(mode: SyncTransportMode): void;
 	setPullInterval(interval: number): void;
 	readonly pullInterval: number;
 	uploadFile(data: FileData): Promise<FileUploadResult>;
 	getFile(fileId: string): Promise<FilePullResult>;
 	readonly presence: PresenceManager<Profile, Presence>;
+	send(message: ClientMessage): void;
+	start(): void;
+	stop(): void;
+	ignoreIncoming(): void;
+	destroy(): void;
+	reconnect(): void;
+	readonly isConnected: boolean;
+	readonly status: 'active' | 'paused';
+	readonly mode: SyncTransportMode;
 }
 
 export class NoSync<Presence = any, Profile = any>
@@ -193,6 +198,7 @@ export class ServerSync<Presence = any, Profile = any>
 		{
 			authEndpoint,
 			fetchAuth,
+			fetch,
 			initialPresence,
 			automaticTransportSelection = true,
 			autoStart,
@@ -229,6 +235,7 @@ export class ServerSync<Presence = any, Profile = any>
 		this.endpointProvider = new ServerSyncEndpointProvider({
 			authEndpoint,
 			fetchAuth,
+			fetch,
 		});
 
 		this.webSocketSync = new WebSocketSync({
@@ -243,6 +250,7 @@ export class ServerSync<Presence = any, Profile = any>
 			presence: this.presence,
 			log: this.log,
 			interval: pullInterval,
+			fetch,
 		});
 		this.fileSync = new FileSync({
 			endpointProvider: this.endpointProvider,