@verdant-web/store 3.12.0 → 4.0.0-next.0

package/src/files/FileManager.ts

@@ -1,184 +1,48 @@
-import { FileData, FileRef } from '@verdant-web/common';
-import { Context } from '../context.js';
-import { Metadata } from '../metadata/Metadata.js';
+import { FileData } from '@verdant-web/common';
+import { Context } from '../context/context.js';
 import { Sync } from '../sync/Sync.js';
-import {
-	EntityFile,
-	MARK_FAILED,
-	MARK_UPLOADED,
-	UPDATE,
-} from './EntityFile.js';
-import {
-	FileStorage,
-	ReturnedFileData,
-	StoredFileData,
-} from './FileStorage.js';
-
-/**
- * Default: if file was deleted > 3 days ago
- */
-function defaultCanCleanup(fileData: ReturnedFileData) {
-	return (
-		fileData.deletedAt !== null &&
-		fileData.deletedAt < Date.now() - 1000 * 60 * 24 * 3
-	);
-}
-
-export interface FileManagerConfig {
-	/**
-	 * Override the heuristic for deciding when a deleted file can be cleaned up.
-	 * By default this waits 3 days since deletion, then deletes the file data.
-	 * If the file has been synchronized to a server, it could still be restored
-	 * if the server has not yet deleted it.
-	 */
-	canCleanupDeletedFile?: (file: ReturnedFileData) => boolean;
-}
+import { EntityFile, MARK_FAILED, UPDATE } from './EntityFile.js';
 
 export class FileManager {
-	private storage;
 	private sync;
 	private context;
 
-	private files = new Map<string, EntityFile>();
-	private config: Required<FileManagerConfig>;
-	private meta: Metadata;
+	private cache = new Map<string, EntityFile>();
 
-	private maxUploadRetries = 3;
-	private maxDownloadRetries = 3;
-
-	constructor({
-		db,
-		sync,
-		context,
-		meta,
-		config = {},
-	}: {
-		db: IDBDatabase;
-		sync: Sync;
-		context: Context;
-		config?: FileManagerConfig;
-		meta: Metadata;
-	}) {
-		this.storage = new FileStorage(db);
+	constructor({ sync, context }: { sync: Sync; context: Context }) {
 		this.sync = sync;
 		this.context = context;
-		this.meta = meta;
-		this.config = {
-			canCleanupDeletedFile: defaultCanCleanup,
-			...config,
-		};
-
-		this.sync.subscribe('onlineChange', this.onOnlineChange);
-		this.meta.subscribe('filesDeleted', this.handleFileRefsDeleted);
-		this.sync.subscribe('serverReset', this.storage.resetSyncedStatusSince);
-		// check on startup to see if files can be cleaned up
-		this.tryCleanupDeletedFiles();
 	}
 
-	add = async (file: FileData) => {
-		// this method accepts a FileData which refers to a remote
-		// file, as well as local files. in the case of a remote file,
-		// we actually re-download and upload the file again. this powers
-		// the cloning of documents with files; we clone their filedata
-		// and re-upload to a new file ID. otherwise, when the cloned
-		// filedata was marked deleted, the original file would be deleted
-		// and the clone would refer to a missing file.
-		if (file.url && !file.file) {
-			const blob = await this.downloadRemoteFile(file.url);
-			// convert blob to file with name and type
-			file.file = new File([blob], file.name, { type: file.type });
-		}
-
-		file.remote = false;
+	add = async (file: FileData, options?: { downloadRemote: boolean }) => {
 		// immediately cache the file
-		if (!this.files.has(file.id)) {
-			const entityFile = new EntityFile(file.id);
+		if (!this.cache.has(file.id)) {
+			const entityFile = new EntityFile(file.id, { ctx: this.context });
 			entityFile[UPDATE](file);
-			this.files.set(file.id, entityFile);
+			this.cache.set(file.id, entityFile);
 		} else {
-			this.files.get(file.id)![UPDATE](file);
-		}
-		// write to local storage and send to sync immediately
-		await this.storage.addFile(file);
-		// send to sync
-		if (file.file && this.sync.status === 'active') {
-			await this.uploadFile(file);
+			this.cache.get(file.id)![UPDATE](file);
 		}
-	};
 
-	private uploadFile = async (file: FileData, retries = 0) => {
-		const result = await this.sync.uploadFile(file);
-		if (result.success) {
-			await this.storage.markUploaded(file.id);
-			const cached = this.files.get(file.id);
-			if (cached) {
-				cached[MARK_UPLOADED]();
-			}
-			this.context.log('info', 'File uploaded', file.id);
-		} else {
-			if (result.retry && retries < this.maxUploadRetries) {
-				this.context.log(
-					'error',
-					`Error uploading file ${file.id}, retrying...`,
-					result.error,
-				);
-				// schedule a retry
-				setTimeout(this.uploadFile, 1000, file, retries + 1);
-			} else {
-				this.context.log(
-					'error',
-					`Failed to upload file ${file.id}. Not retrying until next sync.`,
-					result.error,
-				);
-			}
-		}
-	};
-
-	private downloadRemoteFile = async (
-		url: string,
-		retries = 0,
-	): Promise<Blob> => {
-		const resp = await fetch(url, {
-			method: 'GET',
-			credentials: 'include',
-		});
-		if (!resp.ok) {
-			if (retries < this.maxDownloadRetries) {
-				return new Promise((resolve, reject) => {
-					setTimeout(() => {
-						this.downloadRemoteFile(url, retries + 1).then(resolve, reject);
-					}, 1000);
-				});
-			} else {
-				throw new Error(`Failed to download file: ${resp.status}`);
-			}
-		}
-		const blob = await resp.blob();
-		return blob;
+		await this.context.files.add(file, options);
 	};
 
 	/**
 	 * Immediately returns an EntityFile to use, then either loads
 	 * the file from cache, local database, or the server.
 	 */
-	get = (id: string, options?: { downloadRemote?: boolean }) => {
-		if (this.files.has(id)) {
-			return this.files.get(id)!;
+	get = (id: string, options: { downloadRemote?: boolean; ctx: Context }) => {
+		if (this.cache.has(id)) {
+			return this.cache.get(id)!;
 		}
 		const file = new EntityFile(id, options);
-		this.files.set(id, file);
+		this.cache.set(id, file);
 		this.load(file);
 		return file;
 	};
 
-	private load = async (file: EntityFile, retries = 0) => {
-		if (retries > this.maxDownloadRetries) {
-			this.context.log('error', 'Failed to load file after 5 retries');
-			file[MARK_FAILED]();
-			return;
-		}
-
-		const fileData = await this.storage.getFile(file.id);
+	private load = async (file: EntityFile) => {
+		const fileData = await this.context.files.get(file.id);
 		if (fileData) {
 			file[UPDATE](fileData);
 		} else {
@@ -187,108 +51,17 @@ export class FileManager {
 				const result = await this.sync.getFile(file.id);
 				if (result.success) {
 					file[UPDATE](result.data);
-					await this.storage.addFile(result.data, {
+					await this.context.files.add(result.data, {
 						downloadRemote: file.downloadRemote,
 					});
 				} else {
 					this.context.log('error', 'Failed to load file', result);
 					file[MARK_FAILED]();
-					if (result.retry) {
-						// schedule a retry
-						setTimeout(this.load, 1000, file, retries + 1);
-					}
 				}
 			} catch (err) {
 				this.context.log('error', 'Failed to load file', err);
 				file[MARK_FAILED]();
-				// schedule a retry
-				setTimeout(this.load, 1000, file, retries + 1);
 			}
 		}
 	};
-
-	listUnsynced = async () => {
-		return this.storage.listUnsynced();
-	};
-
-	exportAll = async (downloadRemote = false) => {
-		const storedFiles = await this.storage.getAll();
-		if (downloadRemote) {
-			for (const storedFile of storedFiles) {
-				// if it doesn't have a buffer, we need to read
-				// one from the server
-				if (!storedFile.file && storedFile.url) {
-					try {
-						const blob = await fetch(storedFile.url, {
-							method: 'GET',
-							credentials: 'include',
-						}).then((r) => r.blob());
-						storedFile.file = blob;
-					} catch (err) {
-						this.context.log(
-							'error',
-							"Failed to download file to cache it locally. The file will still be available using its URL. Check the file server's CORS configuration.",
-							err,
-						);
-					}
-				}
-			}
-		}
-		return storedFiles;
-	};
-
-	importAll = async (files: ReturnedFileData[]) => {
-		await Promise.all(files.map((file) => this.add(file)));
-	};
-
-	private onOnlineChange = async (online: boolean) => {
-		// if online, try to upload any unsynced files
-		if (online) {
-			const unsynced = await this.listUnsynced();
-			await Promise.all(unsynced.map(this.uploadFile));
-		}
-	};
-
-	tryCleanupDeletedFiles = async () => {
-		let count = 0;
-		let skipCount = 0;
-		await this.storage.iterateOverPendingDelete((fileData, store) => {
-			if (this.config.canCleanupDeletedFile(fileData)) {
-				count++;
-				store.delete(fileData.id);
-			} else {
-				skipCount++;
-			}
-		});
-
-		this.context.log(
-			'info',
-			`Cleaned up ${count} files, skipped ${skipCount} files`,
-		);
-	};
-
-	private handleFileRefsDeleted = async (fileRefs: FileRef[]) => {
-		const tx = this.storage.createTransaction(['files'], { mode: 'readwrite' });
-		await Promise.all(
-			fileRefs.map(async (fileRef) => {
-				try {
-					await this.storage.markPendingDelete(fileRef.id, { transaction: tx });
-				} catch (err) {
-					this.context.log('error', 'Failed to mark file for deletion', err);
-				}
-			}),
-		);
-		this.context.log(
-			'info',
-			`Marked ${fileRefs.length} files as pending delete`,
-		);
-	};
-
-	close = () => {
-		this.storage.dispose();
-	};
-
-	stats = () => {
-		return this.storage.stats();
-	};
 }

package/src/files/utils.ts

@@ -55,18 +55,3 @@ export function processValueFiles(
 
 	return value;
 }
-
-export function fileToArrayBuffer(file: File | Blob) {
-	// special case for testing...
-	if ('__testReadBuffer' in file) {
-		return file.__testReadBuffer;
-	}
-	return new Promise<ArrayBuffer>((resolve, reject) => {
-		const reader = new FileReader();
-		reader.onload = () => {
-			resolve(reader.result as ArrayBuffer);
-		};
-		reader.onerror = reject;
-		reader.readAsArrayBuffer(file);
-	});
-}

package/src/index.ts

@@ -48,12 +48,13 @@ export type {
 	StorageFieldsSchema,
 	IndexValueTag,
 	Migration,
+	VerdantError,
+	VerdantErrorCode,
 } from '@verdant-web/common';
 export type { UserInfo } from '@verdant-web/common';
 export type { Query } from './queries/types.js';
 export type { QueryStatus } from './queries/BaseQuery.js';
 export type { CollectionQueries } from './queries/CollectionQueries.js';
-export { MigrationPathError } from './migration/errors.js';
 export * from './utils/id.js';
 export { UndoHistory } from './UndoHistory.js';
 export * from './authorization.js';

package/src/{metadata → persistence}/MessageCreator.ts

@@ -8,53 +8,38 @@ import {
 	OperationMessage,
 	pickValidOperationKeys,
 	PresenceUpdateMessage,
-	SyncAckMessage,
 	SyncMessage,
 	VerdantInternalPresence,
 } from '@verdant-web/common';
 
-import { Metadata } from './Metadata.js';
+import { Context } from '../context/context.js';
+import { PersistenceMetadataDb } from './interfaces.js';
 
 export class MessageCreator {
-	constructor(private meta: Metadata) {}
+	constructor(
+		private db: PersistenceMetadataDb,
+		private ctx: Pick<Context, 'time' | 'schema' | 'log'>,
+	) {}
 
 	createOperation = async (
 		init: Pick<OperationMessage, 'operations'> & {
 			timestamp?: string;
 		},
 	): Promise<OperationMessage> => {
-		const localInfo = await this.meta.localReplica.get();
+		const localInfo = await this.db.getLocalReplica();
 		return {
 			type: 'op',
-			timestamp: this.meta.now,
+			timestamp: this.ctx.time.now,
 			replicaId: localInfo.id,
 			operations: init.operations.map(pickValidOperationKeys),
 		};
 	};
 
-	createMigrationOperation = async ({
-		targetVersion,
-		...init
-	}: Pick<OperationMessage, 'operations'> & {
-		targetVersion: number;
-	}): Promise<OperationMessage> => {
-		const localInfo = await this.meta.localReplica.get();
-		return {
-			type: 'op',
-			operations: init.operations.map((op) => ({
-				...op,
-				timestamp: this.meta.time.zero(targetVersion),
-			})),
-			timestamp: this.meta.time.zero(targetVersion),
-			replicaId: localInfo.id,
-		};
-	};
-
 	/**
 	 * @param since - override local understanding of last sync time
 	 */
 	createSyncStep1 = async (since?: string | null): Promise<SyncMessage> => {
-		const localReplicaInfo = await this.meta.localReplica.get();
+		const localReplicaInfo = await this.db.getLocalReplica();
 
 		const provideChangesSince =
 			since === null ? null : localReplicaInfo.lastSyncedLogicalTime;
@@ -64,12 +49,22 @@ export class MessageCreator {
 		const operations: Operation[] = [];
 		const affectedDocs = new Set<ObjectIdentifier>();
 
+		const tx = await this.db.transaction({
+			mode: 'readwrite',
+			storeNames: ['operations', 'baselines'],
+		});
+
 		// FIXME: this branch gives bad vibes. should we always
 		// send all operations from other replicas too? is there
 		// ever a case where we have a "since" timestamp and there
 		// are foreign ops that match it?
 		if (provideChangesSince) {
-			await this.meta.operations.iterateOverAllLocalOperations(
+			this.ctx.log(
+				'debug',
+				'Syncing local operations since',
+				provideChangesSince,
+			);
+			await this.db.iterateLocalOperations(
 				(patch) => {
 					operations.push(pickValidOperationKeys(patch));
 					affectedDocs.add(getOidRoot(patch.oid));
@@ -77,32 +72,47 @@ export class MessageCreator {
 				{
 					after: provideChangesSince,
 					// block on writes to prevent race conditions
-					mode: 'readwrite',
+					transaction: tx,
 				},
 			);
 		} else {
+			this.ctx.log('debug', 'Syncing all operations');
 			// if providing the whole history, don't limit to only local
 			// operations
-			await this.meta.operations.iterateOverAllOperations(
+			await this.db.iterateAllOperations(
 				(patch) => {
 					operations.push(pickValidOperationKeys(patch));
 					affectedDocs.add(getOidRoot(patch.oid));
 				},
 				{
-					mode: 'readwrite',
+					transaction: tx,
 				},
 			);
 		}
 		// we only need to send baselines if we've never synced before
 		let baselines: DocumentBaseline[] = [];
 		if (!provideChangesSince) {
-			baselines = await this.meta.baselines.getAllSince('');
+			await this.db.iterateAllBaselines(
+				(b) => {
+					baselines.push(b);
+				},
+				{
+					transaction: tx,
+				},
+			);
+		}
+
+		if (operations.length > 0) {
+			this.ctx.log(
+				'debug',
+				`Syncing ${operations.length} operations since ${provideChangesSince}`,
+			);
 		}
 
 		return {
 			type: 'sync',
-			schemaVersion: this.meta.schema.currentVersion,
-			timestamp: this.meta.now,
+			schemaVersion: this.ctx.schema.version,
+			timestamp: this.ctx.time.now,
 			replicaId: localReplicaInfo.id,
 			resyncAll: !localReplicaInfo.lastSyncedLogicalTime,
 			operations,
@@ -115,7 +125,7 @@ export class MessageCreator {
 		presence?: any;
 		internal?: VerdantInternalPresence;
 	}): Promise<PresenceUpdateMessage> => {
-		const localReplicaInfo = await this.meta.localReplica.get();
+		const localReplicaInfo = await this.db.getLocalReplica();
 		return {
 			type: 'presence-update',
 			presence: data.presence,
@@ -125,19 +135,19 @@ export class MessageCreator {
 	};
 
 	createHeartbeat = async (): Promise<HeartbeatMessage> => {
-		const localReplicaInfo = await this.meta.localReplica.get();
+		const localReplicaInfo = await this.db.getLocalReplica();
 		return {
 			type: 'heartbeat',
-			timestamp: this.meta.now,
+			timestamp: this.ctx.time.now,
 			replicaId: localReplicaInfo.id,
 		};
 	};
 
 	createAck = async (nonce: string): Promise<AckMessage> => {
-		const localReplicaInfo = await this.meta.localReplica.get();
+		const localReplicaInfo = await this.db.getLocalReplica();
 		return {
 			type: 'ack',
-			timestamp: this.meta.now,
+			timestamp: this.ctx.time.now,
 			replicaId: localReplicaInfo.id,
 			nonce,
 		};