@verdant-web/store 2.0.1

package/src/files/FileManager.ts

@@ -0,0 +1,195 @@
+import { FileData, FileRef } from '@verdant-web/common';
+import { Context } from '../context.js';
+import { Metadata } from '../metadata/Metadata.js';
+import { Sync } from '../sync/Sync.js';
+import { EntityFile, MARK_FAILED, UPDATE } from './EntityFile.js';
+import { FileStorage, ReturnedFileData } from './FileStorage.js';
+
+/**
+ * Default: if file was deleted > 3 days ago
+ */
+function defaultCanCleanup(fileData: ReturnedFileData) {
+	return (
+		fileData.deletedAt !== null &&
+		fileData.deletedAt < Date.now() - 1000 * 60 * 24 * 3
+	);
+}
+
+export interface FileManagerConfig {
+	/**
+	 * Override the heuristic for deciding when a deleted file can be cleaned up.
+	 * By default this waits 3 days since deletion, then deletes the file data.
+	 * If the file has been synchronized to a server, it could still be restored
+	 * if the server has not yet deleted it.
+	 */
+	canCleanupDeletedFile?: (file: ReturnedFileData) => boolean;
+}
+
+export class FileManager {
+	private storage;
+	private sync;
+	private context;
+
+	private files = new Map<string, EntityFile>();
+	private config: Required<FileManagerConfig>;
+	private meta: Metadata;
+
+	constructor({
+		db,
+		sync,
+		context,
+		meta,
+		config = {},
+	}: {
+		db: IDBDatabase;
+		sync: Sync;
+		context: Context;
+		config?: FileManagerConfig;
+		meta: Metadata;
+	}) {
+		this.storage = new FileStorage(db);
+		this.sync = sync;
+		this.context = context;
+		this.meta = meta;
+		this.config = {
+			canCleanupDeletedFile: defaultCanCleanup,
+			...config,
+		};
+
+		this.sync.subscribe('onlineChange', this.onOnlineChange);
+		this.meta.subscribe('filesDeleted', this.handleFileRefsDeleted);
+		// check on startup to see if files can be cleaned up
+		this.tryCleanupDeletedFiles();
+	}
+
+	add = async (fileInput: Omit<FileData, 'remote'>) => {
+		const file = fileInput as unknown as FileData;
+		file.remote = false;
+		// immediately cache the file
+		if (!this.files.has(file.id)) {
+			const entityFile = new EntityFile(file.id);
+			entityFile[UPDATE](file);
+			this.files.set(file.id, entityFile);
+		} else {
+			this.files.get(file.id)![UPDATE](file);
+		}
+		// write to local storage and send to sync immediately
+		await this.storage.addFile(file);
+		// send to sync
+		if (file.file) {
+			const result = await this.sync.uploadFile(file);
+			if (result.success) {
+				await this.storage.markUploaded(file.id);
+			} else {
+				this.context.log('error', 'Failed to upload file');
+			}
+		}
+	};
+
+	private uploadFile = async (file: FileData, retries = 0) => {
+		const result = await this.sync.uploadFile(file);
+		if (result.success) {
+			await this.storage.markUploaded(file.id);
+		} else {
+			if (result.retry && retries < 5) {
+				this.context.log('error', 'Error uploading file, retrying...');
+				// schedule a retry
+				setTimeout(this.uploadFile, 1000, file, retries + 1);
+			} else {
+				this.context.log(
+					'error',
+					'Failed to upload file. Not retrying until next sync.',
+				);
+			}
+		}
+	};
+
+	/**
+	 * Immediately returns an EntityFile to use, then either loads
+	 * the file from cache, local database, or the server.
+	 */
+	get = (id: string) => {
+		if (this.files.has(id)) {
+			return this.files.get(id)!;
+		}
+		const file = new EntityFile(id);
+		this.files.set(id, file);
+		this.load(file);
+		return file;
+	};
+
+	private load = async (file: EntityFile, retries = 0) => {
+		if (retries > 5) {
+			file[MARK_FAILED]();
+			return;
+		}
+
+		const fileData = await this.storage.getFile(file.id);
+		if (fileData) {
+			file[UPDATE](fileData);
+		} else {
+			// maybe we don't have it yet, it might be on the server still.
+			try {
+				const result = await this.sync.getFile(file.id);
+				if (result.success) {
+					file[UPDATE](result.data);
+					await this.storage.addFile(result.data);
+				} else {
+					file[MARK_FAILED]();
+					if (result.retry) {
+						// schedule a retry
+						setTimeout(this.load, 1000, file, retries + 1);
+					}
+				}
+			} catch (err) {
+				this.context.log('error', 'Failed to load file', err);
+				file[MARK_FAILED]();
+				// schedule a retry
+				setTimeout(this.load, 1000, file, retries + 1);
+			}
+		}
+	};
+
+	listUnsynced = async () => {
+		return this.storage.listUnsynced();
+	};
+
+	private onOnlineChange = async (online: boolean) => {
+		// if online, try to upload any unsynced files
+		if (online) {
+			const unsynced = await this.listUnsynced();
+			await Promise.all(unsynced.map(this.uploadFile));
+		}
+	};
+
+	private tryCleanupDeletedFiles = async () => {
+		let count = 0;
+		let skipCount = 0;
+		await this.storage.iterateOverPendingDelete((fileData, store) => {
+			if (this.config.canCleanupDeletedFile(fileData)) {
+				count++;
+				store.delete(fileData.id);
+			} else {
+				skipCount++;
+			}
+		});
+
+		this.context.log(
+			'info',
+			`Cleaned up ${count} files, skipped ${skipCount} files`,
+		);
+	};
+
+	private handleFileRefsDeleted = async (fileRefs: FileRef[]) => {
+		const tx = this.storage.createTransaction(['files'], 'readwrite');
+		await Promise.all(
+			fileRefs.map(async (fileRef) => {
+				await this.storage.markPendingDelete(fileRef.id, { transaction: tx });
+			}),
+		);
+		this.context.log(
+			'info',
+			`Marked ${fileRefs.length} files as pending delete`,
+		);
+	};
+}

package/src/files/FileStorage.ts

@@ -0,0 +1,177 @@
+import { FileData } from '@verdant-web/common';
+import { IDBService } from '../IDBService.js';
+import { fileToArrayBuffer } from './utils.js';
+
+/**
+ * When stored in IDB, replace the file blob with an array buffer
+ * since it's more compatible, and replace remote boolean with
+ * a string since IDB doesn't support boolean indexes.
+ */
+interface StoredFileData extends Omit<FileData, 'remote' | 'file'> {
+	remote: 'true' | 'false';
+	buffer?: ArrayBuffer;
+	deletedAt: number | null;
+}
+
+export interface ReturnedFileData extends FileData {
+	deletedAt: number | null;
+}
+
+export class FileStorage extends IDBService {
+	addFile = async (
+		file: FileData,
+		{ transaction }: { transaction?: IDBTransaction } = {},
+	) => {
+		const buffer = file.file ? await fileToArrayBuffer(file.file) : undefined;
+		return this.run(
+			'files',
+			(store) => {
+				return store.put({
+					id: file.id,
+					// IDB doesn't support boolean indexes
+					remote: file.remote ? 'true' : 'false',
+					deletedAt: null,
+					name: file.name,
+					type: file.type,
+					url: file.url,
+					buffer,
+				} as StoredFileData);
+			},
+			'readwrite',
+			transaction,
+		);
+	};
+
+	private hydrateFileData = (raw: StoredFileData): ReturnedFileData => {
+		(raw as any).remote = raw.remote === 'true';
+		const buffer = raw.buffer;
+		delete raw.buffer;
+		(raw as unknown as FileData).file = buffer
+			? arrayBufferToBlob(buffer, raw.type)
+			: undefined;
+		return raw as unknown as ReturnedFileData;
+	};
+
+	markUploaded = async (
+		id: string,
+		{ transaction }: { transaction?: IDBTransaction } = {},
+	) => {
+		const current = await this.getFileRaw(id, { transaction });
+
+		if (!current) {
+			throw new Error('File is not in local database');
+		}
+
+		return this.run(
+			'files',
+			(store) => {
+				return store.put({
+					...current,
+					remote: 'true',
+				} as StoredFileData);
+			},
+			'readwrite',
+			transaction,
+		);
+	};
+
+	private getFileRaw = async (
+		id: string,
+		{ transaction }: { transaction?: IDBTransaction } = {},
+	): Promise<StoredFileData | undefined> => {
+		const raw = await this.run<StoredFileData>(
+			'files',
+			(store) => {
+				return store.get(id);
+			},
+			'readonly',
+			transaction,
+		);
+		if (!raw) {
+			return undefined;
+		}
+		return raw;
+	};
+
+	getFile = async (
+		id: string,
+		{ transaction }: { transaction?: IDBTransaction } = {},
+	): Promise<ReturnedFileData | undefined> => {
+		const raw = await this.getFileRaw(id, { transaction });
+		if (!raw) {
+			return undefined;
+		}
+		return this.hydrateFileData(raw);
+	};
+
+	deleteFile(
+		id: string,
+		{ transaction }: { transaction?: IDBTransaction } = {},
+	) {
+		return this.run<undefined>(
+			'files',
+			(store) => {
+				return store.delete(id);
+			},
+			'readwrite',
+			transaction,
+		);
+	}
+
+	markPendingDelete = async (
+		id: string,
+		{ transaction }: { transaction?: IDBTransaction } = {},
+	) => {
+		const current = await this.getFileRaw(id, { transaction });
+
+		if (!current) {
+			throw new Error('File is not in local database');
+		}
+
+		return this.run(
+			'files',
+			(store) => {
+				return store.put({
+					...current,
+					deletedAt: Date.now(),
+				} as StoredFileData);
+			},
+			'readwrite',
+			transaction,
+		);
+	};
+
+	listUnsynced = async () => {
+		const raw = await this.run<StoredFileData[]>(
+			'files',
+			(store) => {
+				return store.index('remote').getAll('false');
+			},
+			'readonly',
+		);
+		return raw.map(this.hydrateFileData);
+	};
+
+	iterateOverPendingDelete = (
+		iterator: (file: ReturnedFileData, store: IDBObjectStore) => void,
+		transaction?: IDBTransaction,
+	) => {
+		return this.iterate<StoredFileData>(
+			'files',
+			(store) => {
+				return store
+					.index('deletedAt')
+					.openCursor(IDBKeyRange.lowerBound(0, true));
+			},
+			(value, store) => {
+				iterator(this.hydrateFileData(value), store);
+			},
+			'readwrite',
+			transaction,
+		);
+	};
+}
+
+export function arrayBufferToBlob(buffer: ArrayBuffer, type: string) {
+	return new Blob([buffer], { type });
+}

package/src/files/utils.test.ts

@@ -0,0 +1,83 @@
+import { FileData } from '@verdant-web/common';
+import { describe, expect, it } from 'vitest';
+import { processValueFiles } from './utils.js';
+
+function makeFile() {
+	return new File([''], 'test.txt', { type: 'text/plain' });
+}
+
+describe('processing files in a value', () => {
+	it('replaces a single file with a reference', () => {
+		const file = makeFile();
+		const list: FileData[] = [];
+		const result = processValueFiles(file, list.push.bind(list));
+		expect(result).toEqual({
+			'@@type': 'file',
+			id: expect.stringContaining(''),
+		});
+		expect(list).toEqual([
+			{
+				id: expect.stringContaining(''),
+				file,
+				url: undefined,
+				remote: false,
+				name: 'test.txt',
+				type: 'text/plain',
+			},
+		]);
+	});
+
+	it('replaces a list of files with references', () => {
+		const file1 = makeFile();
+		const file2 = makeFile();
+		const list: FileData[] = [];
+		const result = processValueFiles([file1, file2], list.push.bind(list));
+		expect(result).toEqual([
+			{ '@@type': 'file', id: expect.stringContaining('') },
+			{ '@@type': 'file', id: expect.stringContaining('') },
+		]);
+		expect(list).toEqual([
+			{
+				id: expect.stringContaining(''),
+				file: file1,
+				url: undefined,
+				remote: false,
+				name: 'test.txt',
+				type: 'text/plain',
+			},
+			{
+				id: expect.stringContaining(''),
+				file: file2,
+				url: undefined,
+				remote: false,
+				name: 'test.txt',
+				type: 'text/plain',
+			},
+		]);
+		expect(result[0].id).not.toEqual(result[1].id);
+	});
+
+	it('replaces nested files with references', () => {
+		const file = makeFile();
+		const list: FileData[] = [];
+		const result = processValueFiles({ a: { b: file } }, list.push.bind(list));
+		expect(result).toEqual({
+			a: {
+				b: {
+					'@@type': 'file',
+					id: expect.stringContaining(''),
+				},
+			},
+		});
+		expect(list).toEqual([
+			{
+				id: expect.stringContaining(''),
+				file,
+				url: undefined,
+				remote: false,
+				name: 'test.txt',
+				type: 'text/plain',
+			},
+		]);
+	});
+});

package/src/files/utils.ts

@@ -0,0 +1,55 @@
+import { createFileRef, FileData } from '@verdant-web/common';
+import cuid from 'cuid';
+
+export function createFileData(file: File): FileData {
+	return {
+		id: cuid(),
+		file: file,
+		url: undefined,
+		remote: false,
+		name: file.name,
+		type: file.type,
+	};
+}
+
+/**
+ * Replaces File values with refs and returns the normalized value.
+ * The list of files passed to the second argument will be populated with the files found in the value.
+ */
+export function processValueFiles(
+	value: any,
+	onFileIdentified: (fileData: FileData) => void,
+): any {
+	if (value instanceof File) {
+		const data = createFileData(value);
+		onFileIdentified(data);
+		return createFileRef(data.id);
+	}
+
+	if (Array.isArray(value)) {
+		for (let i = 0; i < value.length; i++) {
+			value[i] = processValueFiles(value[i], onFileIdentified);
+		}
+		return value;
+	}
+
+	if (typeof value === 'object') {
+		for (const key in value) {
+			value[key] = processValueFiles(value[key], onFileIdentified);
+		}
+		return value;
+	}
+
+	return value;
+}
+
+export function fileToArrayBuffer(file: File | Blob) {
+	return new Promise<ArrayBuffer>((resolve, reject) => {
+		const reader = new FileReader();
+		reader.onload = () => {
+			resolve(reader.result as ArrayBuffer);
+		};
+		reader.onerror = reject;
+		reader.readAsArrayBuffer(file);
+	});
+}

package/src/idb.ts

@@ -0,0 +1,88 @@
+import { roughSizeOfObject } from '@verdant-web/common';
+
+export function storeRequestPromise<T>(request: IDBRequest<T>) {
+	return new Promise<T>((resolve, reject) => {
+		request.onsuccess = () => {
+			resolve(request.result);
+		};
+		request.onerror = () => {
+			reject(request.error);
+		};
+	});
+}
+
+export function cursorIterator<T>(
+	request: IDBRequest<IDBCursorWithValue | null>,
+	callback: (value: T | null) => boolean,
+) {
+	return new Promise<void>((resolve, reject) => {
+		request.onsuccess = () => {
+			const cursor = request.result;
+			if (cursor) {
+				if (callback(cursor.value)) {
+					cursor.continue();
+				} else {
+					resolve();
+				}
+			} else {
+				resolve();
+			}
+		};
+		request.onerror = () => {
+			reject(request.error);
+		};
+	});
+}
+
+export function getSizeOfObjectStore(
+	database: IDBDatabase,
+	storeName: string,
+): Promise<{ count: number; size: number }> {
+	return new Promise((resolve, reject) => {
+		const tx = database.transaction([storeName], 'readonly');
+		const store = tx.objectStore(storeName);
+		const cursorReq = store.openCursor();
+		let count = 0;
+		let size = 0;
+		cursorReq.onsuccess = function (e) {
+			const cursor = cursorReq.result;
+			if (cursor) {
+				count++;
+				size = size + roughSizeOfObject(cursor.value);
+				cursor.continue();
+			}
+		};
+		cursorReq.onerror = function (e) {
+			reject(e);
+		};
+		tx.oncomplete = function (e) {
+			resolve({
+				count: count,
+				size: size,
+			});
+		};
+		tx.onabort = function (e) {
+			reject(e);
+		};
+		tx.onerror = function (e) {
+			reject(e);
+		};
+	});
+}
+
+export function getAllFromObjectStores(db: IDBDatabase, stores: string[]) {
+	const transaction = db.transaction(stores, 'readonly');
+	const promises = stores.map((store) => {
+		const objectStore = transaction.objectStore(store);
+		return storeRequestPromise(objectStore.getAll());
+	});
+	return Promise.all(promises);
+}
+
+export async function closeDatabase(db: IDBDatabase) {
+	db.close();
+	// wait for microtask queue to clear
+	await new Promise<void>((resolve, reject) => {
+		resolve();
+	});
+}

package/src/index.ts

@@ -0,0 +1,40 @@
+import {
+	ClientDescriptor,
+	ClientDescriptorOptions,
+} from './client/ClientDescriptor.js';
+import { Client } from './client/Client.js';
+export type { ClientWithCollections } from './client/Client.js';
+export { ClientDescriptor };
+export { Client };
+// backward compat
+export { ClientDescriptor as StorageDescriptor };
+export { Client as Storage };
+export type { ClientDescriptorOptions };
+export type { ClientDescriptorOptions as StorageInitOptions };
+export { Entity } from './reactives/Entity.js';
+export type {
+	ObjectEntity,
+	ListEntity,
+	EntityShape,
+	AccessibleEntityProperty,
+	AnyEntity,
+	EntityDestructured,
+} from './reactives/Entity.js';
+export { ServerSync } from './sync/Sync.js';
+export type { SyncTransportMode } from './sync/Sync.js';
+export { EntityFile } from './files/EntityFile.js';
+export {
+	collection,
+	schema,
+	createDefaultMigration,
+	migrate,
+} from '@verdant-web/common';
+export type {
+	StorageDocument,
+	StorageSchema,
+	StorageCollectionSchema,
+	Migration,
+} from '@verdant-web/common';
+export type { UserInfo } from '@verdant-web/common';
+export type { Query } from './queries2/types.js';
+export type { CollectionQueries } from './queries2/CollectionQueries.js';

package/src/indexes.ts

@@ -0,0 +1,31 @@
+import {
+	CollectionCompoundIndex,
+	CompoundIndexValue,
+	createCompoundIndexValue,
+	StorageCollectionSchema,
+	StorageSyntheticIndexSchema,
+} from '@verdant-web/common';
+
+export function computeSynthetics(schema: StorageCollectionSchema, obj: any) {
+	const result: Record<string, any> = {};
+	for (const [name, property] of Object.entries(schema.synthetics || {})) {
+		result[name] = (property as StorageSyntheticIndexSchema<any>).compute(obj);
+	}
+	return result;
+}
+
+export function computeCompoundIndices(
+	schema: StorageCollectionSchema<any, any, any>,
+	doc: any,
+): any {
+	return Object.entries(schema.compounds || {}).reduce<
+		Record<string, CompoundIndexValue>
+	>((acc, [indexKey, index]) => {
+		acc[indexKey] = createCompoundIndexValue(
+			...(index as CollectionCompoundIndex<any, any>).of.map(
+				(key) => doc[key] as string | number,
+			),
+		);
+		return acc;
+	}, {} as Record<string, CompoundIndexValue>);
+}

package/src/metadata/AckInfoStore.ts

@@ -0,0 +1,30 @@
+import { IDBService } from '../IDBService.js';
+
+type AckInfo = {
+	type: 'ack';
+	// null means no operations are acknowledged
+	// by every peer yet.
+	globalAckTimestamp: string | null;
+};
+
+export class AckInfoStore extends IDBService {
+	getAckInfo = async (): Promise<AckInfo> => {
+		const result = await this.run<AckInfo>('info', (store) => store.get('ack'));
+		if (result) {
+			return result;
+		} else {
+			return {
+				globalAckTimestamp: null,
+				type: 'ack',
+			};
+		}
+	};
+
+	setGlobalAck = async (ack: string) => {
+		await this.run(
+			'info',
+			(store) => store.put({ type: 'ack', globalAckTimestamp: ack }),
+			'readwrite',
+		);
+	};
+}