@verdant-web/store 3.12.0 → 4.0.0-next.0

package/src/persistence/PersistenceFiles.ts

@@ -0,0 +1,227 @@
+import { FileData, FileRef } from '@verdant-web/common';
+import { Context, FileConfig } from '../context/context.js';
+import { PersistedFileData, PersistenceFileDb } from './interfaces.js';
+import { Disposable } from '../utils/Disposable.js';
+
+export class PersistenceFiles extends Disposable {
+	constructor(
+		private db: PersistenceFileDb,
+		private context: Omit<Context, 'queries'>,
+	) {
+		super();
+		context.internalEvents.subscribe('filesDeleted', this.onFileRefsDeleted);
+		this.compose(this.db);
+		// on startup, try deleting old files.
+		this.cleanupDeletedFiles();
+	}
+
+	private get config(): Required<FileConfig> {
+		return {
+			canCleanupDeletedFile(fileData) {
+				return (
+					fileData.deletedAt !== null &&
+					fileData.deletedAt < Date.now() - 1000 * 60 * 24 * 3
+				);
+			},
+			...this.context.config.files,
+		};
+	}
+
+	onServerReset = (since: string | null) =>
+		this.db.resetSyncedStatusSince(since);
+	add = async (file: FileData, options?: { downloadRemote?: boolean }) => {
+		// this method accepts a FileData which refers to a remote
+		// file, as well as local files. in the case of a remote file,
+		// we actually re-download and upload the file again. this powers
+		// the cloning of documents with files; we clone their filedata
+		// and re-upload to a new file ID. otherwise, when the cloned
+		// filedata was marked deleted, the original file would be deleted
+		// and the clone would refer to a missing file.
+		if (file.url && !file.file) {
+			this.context.log(
+				'debug',
+				'Remote file added to an entity. This usually means an entity was cloned. Downloading remote file...',
+				file.id,
+			);
+			const blob = await this.context.files.downloadRemoteFile(file.url, 0, 3);
+			// convert blob to file with name and type
+			file.file = new File([blob], file.name, { type: file.type });
+		} else if (!file.file) {
+			this.context.log(
+				'warn',
+				'File added without a file or URL. This file will not be available for use.',
+				file.id,
+			);
+		}
+
+		file.remote = false;
+		// fire event for processing immediately
+		this.context.internalEvents.emit('fileAdded', file);
+		// store in persistence db
+		await this.db.add(file, options);
+		this.context.log(
+			'debug',
+			'File added',
+			file.id,
+			file.name,
+			file.type,
+			file.file ? 'with binary file' : file.url ? 'with url' : 'with no data',
+		);
+	};
+	onUploaded = this.db.markUploaded.bind(this.db);
+	get = this.db.get.bind(this.db);
+	getAll = this.db.getAll.bind(this.db);
+	listUnsynced = this.db.listUnsynced.bind(this.db);
+	iterateOverPendingDelete = this.db.iterateOverPendingDelete.bind(this.db);
+	stats = this.db.stats.bind(this.db);
+
+	private getFileExportName = (originalFileName: string, id: string) => {
+		return `${id}___${originalFileName}`;
+	};
+	export = async (downloadRemote = false) => {
+		const storedFiles = await this.getAll();
+		if (downloadRemote) {
+			for (const storedFile of storedFiles) {
+				// if it doesn't have a buffer, we need to read one from the server
+				if (!storedFile.file && storedFile.url) {
+					try {
+						const blob = await this.downloadRemoteFile(storedFile.url);
+						storedFile.file = blob;
+					} catch (err) {
+						this.context.log(
+							'error',
+							"Failed to download file to cache it locally. The file will still be available using its URL. Check the file server's CORS configuration.",
+							err,
+						);
+					}
+				}
+			}
+		}
+		// split files into data and files
+		const fileData: Array<Omit<PersistedFileData, 'file'>> = [];
+		const files: Array<File> = [];
+
+		for (const fileExport of storedFiles) {
+			const file = fileExport.file;
+			delete fileExport.file;
+			fileData.push(fileExport);
+			if (file) {
+				// rename with ID
+				const asFile = new File(
+					[file],
+					this.getFileExportName(fileExport.name, fileExport.id),
+					{
+						type: fileExport.type,
+					},
+				);
+				files.push(asFile);
+			} else {
+				this.context.log(
+					'warn',
+					`File ${fileExport.id} was could not be loaded locally or from the server. It will be missing in the export.`,
+				);
+			}
+		}
+		return {
+			fileData,
+			files,
+		};
+	};
+
+	import = async ({
+		fileData,
+		files,
+	}: {
+		fileData: Array<Omit<PersistedFileData, 'file'>>;
+		files: File[];
+	}) => {
+		// re-attach files to their file data and import
+		const fileToIdMap = new Map(
+			files.map((file) => {
+				const { id } = this.parseFileExportname(file.name);
+				return [id, file];
+			}),
+		);
+		const importedFiles: PersistedFileData[] = fileData.map((fileData) => {
+			const file = fileToIdMap.get(fileData.id);
+
+			if (!file) {
+				this.context.log('warn', `File ${fileData.id} was not found in import`);
+				return fileData;
+			}
+
+			return {
+				...fileData,
+				file,
+			};
+		});
+		await Promise.all(importedFiles.map((file) => this.add(file)));
+	};
+
+	private parseFileExportname = (name: string) => {
+		const [id, originalFileName] = name.split('___');
+		return { id, originalFileName };
+	};
+
+	downloadRemoteFile = async (url: string, retries = 0, maxRetries = 0) => {
+		const resp = await fetch(url, {
+			method: 'GET',
+			credentials: 'include',
+		});
+		if (!resp.ok) {
+			if (retries < maxRetries) {
+				return new Promise<Blob>((resolve, reject) => {
+					setTimeout(() => {
+						this.downloadRemoteFile(url, retries + 1, maxRetries).then(
+							resolve,
+							reject,
+						);
+					}, 1000);
+				});
+			} else {
+				throw new Error(
+					`Failed to download file after ${maxRetries} retries (status: ${resp.status})`,
+				);
+			}
+		}
+		return await resp.blob();
+	};
+
+	cleanupDeletedFiles = async () => {
+		let count = 0;
+		let skipCount = 0;
+		await this.iterateOverPendingDelete((fileData, store) => {
+			if (this.config.canCleanupDeletedFile(fileData)) {
+				count++;
+				store.delete(fileData.id);
+			} else {
+				skipCount++;
+			}
+		});
+
+		this.context.log(
+			'info',
+			`Cleaned up ${count} files, skipped ${skipCount} files`,
+		);
+	};
+
+	private onFileRefsDeleted = async (fileRefs: FileRef[]) => {
+		const tx = this.db.transaction({
+			mode: 'readwrite',
+			storeNames: ['files'],
+		});
+		await Promise.all(
+			fileRefs.map(async (fileRef) => {
+				try {
+					await this.db.markPendingDelete(fileRef.id, { transaction: tx });
+				} catch (err) {
+					this.context.log('error', 'Failed to mark file for deletion', err);
+				}
+			}),
+		);
+		this.context.log(
+			'info',
+			`Marked ${fileRefs.length} files as pending delete`,
+		);
+	};
+}

package/src/persistence/PersistenceMetadata.ts

@@ -0,0 +1,425 @@
+import {
+	applyPatch,
+	assert,
+	assignOid,
+	ClientMessage,
+	DocumentBaseline,
+	EventSubscriber,
+	getOidRoot,
+	ObjectIdentifier,
+	Operation,
+	substituteRefsWithObjects,
+} from '@verdant-web/common';
+import {
+	AbstractTransaction,
+	ClientOperation,
+	CommonQueryOptions,
+	MetadataExport,
+	PersistenceMetadataDb,
+} from './interfaces.js';
+import { InitialContext } from '../context/context.js';
+import { PersistenceRebaser } from './PersistenceRebaser.js';
+import { MessageCreator } from './MessageCreator.js';
+import { Disposable } from '../utils/Disposable.js';
+
+export class PersistenceMetadata extends Disposable {
+	private rebaser: PersistenceRebaser;
+	/** Available to others, like sync... */
+	readonly messageCreator: MessageCreator;
+	readonly events = new EventSubscriber<{
+		syncMessage: (message: ClientMessage) => void;
+	}>();
+
+	constructor(
+		private db: PersistenceMetadataDb,
+		private ctx: InitialContext,
+	) {
+		super();
+		this.rebaser = new PersistenceRebaser(db, ctx);
+		this.messageCreator = new MessageCreator(db, ctx);
+		this.compose(this.db);
+	}
+
+	private insertOperations = async (
+		operations: ClientOperation[],
+		options?: { transaction?: AbstractTransaction },
+	) => {
+		this.ctx.log(
+			'debug',
+			`Inserting ${operations.length} operations`,
+			operations,
+		);
+
+		const affectedDocumentOids = await this.db.addOperations(
+			operations,
+			options,
+		);
+
+		for (const op of operations) {
+			this.ctx.globalEvents.emit('operation', op);
+		}
+
+		// we can now enqueue and check for rebase opportunities
+		if (!this.ctx.config.persistence?.disableRebasing) {
+			this.rebaser.tryAutonomousRebase();
+		}
+
+		return affectedDocumentOids;
+	};
+
+	private insertLocalOperations = async (
+		operations: Operation[],
+		options?: { transaction?: AbstractTransaction },
+	) => {
+		if (operations.length === 0) return;
+
+		// add local flag, in place.
+		for (const operation of operations) {
+			(operation as ClientOperation).isLocal = true;
+		}
+		await this.insertOperations(operations as ClientOperation[], options);
+
+		const message = await this.messageCreator.createOperation({ operations });
+		this.events.emit('syncMessage', message);
+	};
+
+	private insertRemoteOperations = async (
+		operations: Operation[],
+		options?: { transaction?: AbstractTransaction },
+	) => {
+		if (operations.length === 0) return [];
+
+		// add local flag, in place
+		for (const operation of operations) {
+			(operation as ClientOperation).isLocal = false;
+		}
+
+		await this.insertOperations(operations as ClientOperation[], options);
+
+		this.ack(operations[operations.length - 1].timestamp);
+	};
+
+	private insertRemoteBaselines = async (
+		baselines: DocumentBaseline[],
+		options?: { transaction?: AbstractTransaction },
+	) => {
+		if (baselines.length === 0) return [];
+		this.ctx.log('debug', `Inserting ${baselines.length} remote baselines`);
+
+		await this.db.setBaselines(baselines, options);
+
+		// this.ack(baselines[baselines.length - 1].timestamp);
+
+		const affectedOidSet = new Set<ObjectIdentifier>();
+		baselines.forEach((baseline) => {
+			affectedOidSet.add(getOidRoot(baseline.oid));
+		});
+
+		return Array.from(affectedOidSet);
+	};
+
+	deleteDocument = async (rootOid: string) => {
+		const oids = new Set<ObjectIdentifier>();
+		const documentOid = getOidRoot(rootOid);
+		assert(documentOid === rootOid, 'Must be root document OID');
+		oids.add(documentOid);
+		// readwrite mode to block on other write transactions
+		const transaction = this.db.transaction({
+			storeNames: ['baselines', 'operations'],
+		});
+		await Promise.all([
+			this.db.iterateDocumentBaselines(
+				documentOid,
+				(baseline) => {
+					oids.add(baseline.oid);
+				},
+				{ transaction },
+			),
+			this.db.iterateDocumentOperations(
+				documentOid,
+				(patch) => {
+					oids.add(patch.oid);
+				},
+				{ transaction },
+			),
+		]);
+		const authz = await this.getDocumentAuthz(documentOid);
+		const ops = new Array<Operation>();
+		for (const oid of oids) {
+			ops.push({
+				oid,
+				timestamp: this.ctx.time.now,
+				data: { op: 'delete' },
+				authz,
+			});
+		}
+		return this.insertLocalOperations(ops);
+	};
+
+	deleteCollection = async (collection: string) => {
+		const oids = new Set<ObjectIdentifier>();
+		const transaction = this.db.transaction({
+			storeNames: ['baselines', 'operations'],
+			mode: 'readwrite',
+		});
+		await Promise.all([
+			this.db.iterateCollectionBaselines(
+				collection,
+				(baseline) => {
+					oids.add(baseline.oid);
+				},
+				{ transaction },
+			),
+			this.db.iterateCollectionOperations(
+				collection,
+				(patch) => {
+					oids.add(patch.oid);
+				},
+				{ transaction },
+			),
+		]);
+
+		const ops = new Array<Operation>();
+		for (const oid of oids) {
+			ops.push({
+				oid,
+				timestamp: this.ctx.time.now,
+				data: { op: 'delete' },
+				authz: undefined,
+			});
+		}
+
+		return this.insertLocalOperations(ops);
+	};
+
+	getDocumentSnapshot = async (
+		oid: ObjectIdentifier,
+		options: { to?: string } = {},
+	) => {
+		const documentOid = getOidRoot(oid);
+		assert(documentOid === oid, 'Must be root document OID');
+		const transaction = this.db.transaction({
+			storeNames: ['baselines', 'operations'],
+			mode: 'readwrite',
+		});
+		const baselines: DocumentBaseline[] = [];
+		await this.db.iterateDocumentBaselines(
+			documentOid,
+			(b) => {
+				baselines.push(b);
+			},
+			{
+				transaction,
+			},
+		);
+		const objectMap = new Map<ObjectIdentifier, any>();
+		for (const baseline of baselines) {
+			if (baseline.snapshot) {
+				assignOid(baseline.snapshot, baseline.oid);
+			}
+			objectMap.set(baseline.oid, baseline.snapshot);
+		}
+		await this.db.iterateDocumentOperations(
+			documentOid,
+			(op) => {
+				const obj = objectMap.get(op.oid) || undefined;
+				const newObj = applyPatch(obj, op.data);
+				if (newObj) {
+					assignOid(newObj, op.oid);
+				}
+				objectMap.set(op.oid, newObj);
+			},
+			{
+				transaction,
+				// only apply operations up to the current time
+				to: options.to || this.ctx.time.now,
+			},
+		);
+		const root = objectMap.get(documentOid);
+		if (root) {
+			substituteRefsWithObjects(root, objectMap);
+		}
+		return root;
+	};
+
+	getDocumentData = async (
+		oid: ObjectIdentifier,
+		options?: { abort?: AbortSignal },
+	) => {
+		const transaction = this.db.transaction({
+			storeNames: ['baselines', 'operations'],
+			abort: options?.abort,
+		});
+		const baselines: DocumentBaseline[] = [];
+		const operations: Record<ObjectIdentifier, Operation[]> = {};
+		await Promise.all([
+			this.db.iterateDocumentBaselines(
+				oid,
+				(baseline) => {
+					baselines.push(baseline);
+				},
+				{
+					transaction,
+				},
+			),
+			this.db.iterateDocumentOperations(
+				oid,
+				(op) => {
+					operations[op.oid] ??= [];
+					operations[op.oid].push(op);
+				},
+				{ transaction },
+			),
+		]);
+		return {
+			baselines,
+			operations,
+		};
+	};
+
+	getDocumentAuthz = async (oid: ObjectIdentifier) => {
+		let authz;
+		await this.db.iterateEntityOperations(oid, (op) => {
+			if (op.data.op === 'initialize') {
+				authz = op.authz;
+				return true;
+			}
+		});
+		return authz;
+	};
+
+	insertData = async (
+		data: {
+			baselines?: DocumentBaseline[];
+			operations?: Operation[];
+			isLocal?: boolean;
+		},
+		options?: { abort?: AbortSignal },
+	) => {
+		const transaction = this.db.transaction({
+			storeNames: ['baselines', 'operations'],
+			abort: options?.abort,
+			mode: 'readwrite',
+		});
+		if (data.baselines) {
+			await this.insertRemoteBaselines(data.baselines, { transaction });
+		}
+		if (options?.abort?.aborted) return;
+		if (data.operations) {
+			if (data.isLocal) {
+				await this.insertLocalOperations(data.operations, { transaction });
+			} else {
+				await this.insertRemoteOperations(data.operations, { transaction });
+			}
+		}
+	};
+
+	updateLastSynced = async (timestamp: string) => {
+		if (this.ctx.closing) return;
+
+		return this.db.updateLocalReplica({
+			lastSyncedLogicalTime: timestamp,
+		});
+	};
+	setGlobalAck = async (ack: string) => {
+		if (this.ctx.closing) return;
+		await this.db.setGlobalAck(ack);
+		if (!this.ctx.config.persistence?.disableRebasing) {
+			await this.rebaser.scheduleRebase(ack);
+		}
+	};
+
+	getLocalReplica = async (options?: CommonQueryOptions) => {
+		return this.db.getLocalReplica(options);
+	};
+
+	// used to construct sync messages
+	iterateLocalOperations = this.db.iterateLocalOperations;
+	iterateAllOperations = this.db.iterateAllOperations;
+	iterateAllBaselines = this.db.iterateAllBaselines;
+
+	reset = this.db.reset;
+	stats = this.db.stats;
+
+	export = async (): Promise<MetadataExport> => {
+		const db = this.db;
+		const baselines = new Array<DocumentBaseline>();
+		const operations = new Array<ClientOperation>();
+		const transaction = db.transaction({
+			storeNames: ['baselines', 'operations'],
+			mode: 'readwrite',
+		});
+		await this.iterateAllOperations(
+			(op) => {
+				operations.push(op);
+			},
+			{ transaction },
+		);
+		await this.iterateAllBaselines(
+			(baseline) => {
+				baselines.push(baseline);
+			},
+			{ transaction },
+		);
+		const localReplica = await this.db.getLocalReplica();
+		return {
+			operations,
+			baselines,
+			localReplica,
+			schemaVersion: this.ctx.schema.version,
+		};
+	};
+
+	resetFrom = async (data: MetadataExport) => {
+		const db = this.db;
+		const transaction = db.transaction({
+			storeNames: ['baselines', 'operations', 'info'],
+			mode: 'readwrite',
+		});
+		await this.db.reset({ clearReplica: true, transaction });
+		if (data.localReplica) {
+			await this.db.updateLocalReplica(
+				{
+					ackedLogicalTime: data.localReplica.ackedLogicalTime,
+					lastSyncedLogicalTime: data.localReplica.lastSyncedLogicalTime,
+				},
+				{ transaction },
+			);
+		}
+		this.ctx.log('debug', 'Resetting metadata from export', data);
+		await this.insertData({
+			operations: data.operations,
+			baselines: data.baselines,
+			isLocal: true,
+		});
+	};
+
+	manualRebase = async () => {
+		if (this.ctx.closing || this.ctx.config.persistence?.disableRebasing)
+			return;
+		const ackInfo = await this.db.getAckInfo();
+		if (ackInfo.globalAckTimestamp) {
+			await this.rebaser.scheduleRebase(ackInfo.globalAckTimestamp);
+		}
+	};
+
+	private ack = async (timestamp: string) => {
+		const localReplicaInfo = await this.db.getLocalReplica();
+		// can't ack timestamps from the future.
+		if (timestamp > this.ctx.time.now) return;
+
+		this.events.emit('syncMessage', {
+			type: 'ack',
+			replicaId: localReplicaInfo.id,
+			timestamp,
+		});
+
+		if (
+			!this.ctx.closing &&
+			(!localReplicaInfo.ackedLogicalTime ||
+				timestamp > localReplicaInfo.ackedLogicalTime)
+		) {
+			this.db.updateLocalReplica({ ackedLogicalTime: timestamp });
+		}
+	};
+}

package/src/persistence/PersistenceQueries.ts

@@ -0,0 +1,22 @@
+import { PersistenceQueryDb } from './interfaces.js';
+import { Context } from '../context/context.js';
+import { Disposable } from '../utils/Disposable.js';
+
+export class PersistenceQueries extends Disposable {
+	constructor(
+		private db: PersistenceQueryDb,
+		private ctx: Omit<Context, 'queries'>,
+	) {
+		super();
+		this.compose(this.db);
+	}
+
+	reset = this.db.reset.bind(this.db);
+
+	saveEntities = this.db.saveEntities;
+
+	findOneOid = this.db.findOneOid.bind(this.db);
+	findAllOids = this.db.findAllOids.bind(this.db);
+
+	stats = this.db.stats.bind(this.db);
+}