@verdant-web/store 2.8.5 → 3.0.0

package/src/entities/OperationBatcher.ts

@@ -0,0 +1,251 @@
+import {
+	Batcher,
+	Operation,
+	generateId,
+	getOidRoot,
+	getUndoOperations,
+	groupPatchesByOid,
+} from '@verdant-web/common';
+import { Metadata } from '../metadata/Metadata.js';
+import { Context } from '../context.js';
+import type { EntityStore } from './EntityStore.js';
+import { Entity } from './Entity.js';
+
+const DEFAULT_BATCH_KEY = '@@default';
+
+export interface OperationBatch {
+	run: (fn: () => void) => this;
+	/** @deprecated - use commit() */
+	flush: () => Promise<void>;
+	commit: () => Promise<void>;
+	discard: () => void;
+}
+
+export class OperationBatcher {
+	private batcher;
+	private currentBatchKey = DEFAULT_BATCH_KEY;
+	private defaultBatchTimeout: number;
+	private meta;
+	private ctx;
+	private entities;
+
+	constructor({
+		batchTimeout = 200,
+		meta,
+		ctx,
+		entities,
+	}: {
+		batchTimeout?: number;
+		meta: Metadata;
+		ctx: Context;
+		entities: EntityStore;
+	}) {
+		this.meta = meta;
+		this.ctx = ctx;
+		this.entities = entities;
+		this.defaultBatchTimeout = batchTimeout;
+		this.batcher = new Batcher<Operation, { undoable?: boolean }>(
+			this.flushOperations,
+		);
+		this.batcher.add({
+			key: DEFAULT_BATCH_KEY,
+			items: [],
+			max: 100,
+			timeout: batchTimeout,
+			userData: { undoable: true },
+		});
+	}
+
+	get isDefaultBatch() {
+		return this.currentBatchKey === DEFAULT_BATCH_KEY;
+	}
+
+	private flushOperations = async (
+		operations: Operation[],
+		batchKey: string,
+		meta: { undoable?: boolean },
+	) => {
+		this.ctx.log(
+			'debug',
+			'Flushing',
+			operations.length,
+			'operations from batch',
+			batchKey,
+			'to storage / sync',
+		);
+		if (!operations.length) return;
+		// rewrite timestamps of all operations to now - this preserves
+		// the linear history of operations which are sent to the server.
+		// even if multiple batches are spun up in parallel and flushed
+		// after delay, the final operations in each one should reflect
+		// when the batch flushed, not when the changes were made.
+		// This also corresponds to user-observed behavior, since unconfirmed
+		// operations are applied universally after confirmed operations locally,
+		// so even operations which were made before a remote operation but
+		// have not been confirmed yet will appear to come after the remote one
+		// despite the provisional timestamp being earlier
+		// NOTE: this MUST be mutating the original operation object! this timestamp
+		// also serves as a unique ID for deduplication later.
+		for (const op of operations) {
+			op.timestamp = this.meta.now;
+		}
+		await this.commitOperations(operations, meta);
+	};
+
+	/**
+	 * Immediately flushes operations to storage / sync.
+	 * Providing source to second arg skips hydrating related
+	 * Entity from storage, which is useful when that Entity
+	 * isn't in storage (i.e. still creating) or just to speed
+	 * up the commit.
+	 */
+	commitOperations = async (
+		operations: Operation[],
+		meta: { undoable?: boolean; source?: Entity },
+	) => {
+		if (!operations.length) return;
+		// now is the time to decide on what the undo operations will
+		// look like, based on the confirmed view of the related entities.
+		if (meta.undoable) {
+			const undo = await this.createUndo({
+				ops: operations,
+				source: meta.source,
+			});
+			if (undo) this.ctx.undoHistory.addUndo(undo);
+		}
+		// ship it out to EntityStore to compute final snapshots
+		// write to storage and refresh entities and queries
+		await this.entities.addData({
+			operations,
+			baselines: [],
+			isLocal: true,
+		});
+	};
+
+	/**
+	 * Adds operations to the active batch.
+	 */
+	addOperations = (operations: Operation[]) => {
+		if (!operations.length) return;
+		this.batcher.add({
+			key: this.currentBatchKey,
+			items: operations,
+		});
+		this.ctx.log(
+			`debug`,
+			'added',
+			operations.length,
+			'ops to batch',
+			this.currentBatchKey,
+			', size = ',
+			this.batcher.getSize(this.currentBatchKey),
+		);
+	};
+
+	batch = ({
+		undoable = true,
+		batchName = generateId(),
+		max = null,
+		timeout = this.defaultBatchTimeout,
+	}: {
+		undoable?: boolean;
+		batchName?: string;
+		max?: number | null;
+		timeout?: number | null;
+	} = {}): OperationBatch => {
+		const internalBatch = this.batcher.add({
+			key: batchName,
+			max,
+			timeout,
+			items: [],
+			userData: { undoable },
+		});
+		const externalApi = {
+			run: (fn: () => void) => {
+				// while the provided function runs, operations are forwarded
+				// to the new batch instead of default. this relies on the function
+				// being synchronous.
+				this.currentBatchKey = batchName;
+				fn();
+				this.currentBatchKey = DEFAULT_BATCH_KEY;
+				return externalApi;
+			},
+			commit: async () => {
+				// before running a batch, the default operations must be flushed
+				// this better preserves undo history behavior...
+				// if we left the default batch open while flushing a named batch,
+				// then the default batch would be flushed after the named batch,
+				// and the default batch could contain operations both prior and
+				// after the named batch. this would result in a confusing undo
+				// history where the first undo might reverse changes before and
+				// after a set of other changes.
+				await this.batcher.flush(DEFAULT_BATCH_KEY);
+				return internalBatch.flush();
+			},
+			flush: () => externalApi.commit(),
+			discard: () => {
+				this.batcher.discard(batchName);
+			},
+		};
+		return externalApi;
+	};
+
+	flushAll = () => Promise.all(this.batcher.flushAll());
+
+	private createUndo = async (data: { ops: Operation[]; source?: Entity }) => {
+		// this can't be done on-demand because we rely on the current
+		// state of the entities to calculate the inverse operations.
+		const inverseOps = await this.getInverseOperations(data);
+
+		if (!inverseOps.length) return null;
+
+		return async () => {
+			const redo = await this.createUndo({
+				ops: inverseOps,
+				source: data.source,
+			});
+			// set time to now for all undo operations, they're happening now.
+			for (const op of inverseOps) {
+				op.timestamp = this.meta.now;
+			}
+			await this.commitOperations(
+				inverseOps,
+				// undos should not generate their own undo operations
+				// since they already calculate redo as the inverse.
+				{ undoable: false },
+			);
+			return redo;
+		};
+	};
+	private getInverseOperations = async ({
+		ops,
+		source,
+	}: {
+		ops: Operation[];
+		source?: Entity;
+	}) => {
+		const grouped = groupPatchesByOid(ops);
+		const inverseOps: Operation[] = [];
+		const getNow = () => this.meta.now;
+		await Promise.all(
+			Object.entries(grouped).map(async ([oid, patches]): Promise<void> => {
+				const entity = source ?? (await this.entities.hydrate(getOidRoot(oid)));
+				// TODO: this is getting the rebased baseline? how? are ops being submitted early?
+				const viewData = entity?.__getViewData__(oid, 'confirmed');
+				if (!viewData) {
+					this.ctx.log(
+						'warn',
+						'could not find entity',
+						oid,
+						'for undo operation',
+						ops,
+					);
+					return;
+				}
+				const inverse = getUndoOperations(oid, viewData.view, patches, getNow);
+				inverseOps.unshift(...inverse);
+			}),
+		);
+		return inverseOps;
+	};
+}

package/src/entities/types.ts

@@ -0,0 +1,154 @@
+import { ObjectIdentifier } from '@verdant-web/common';
+import type { Entity } from './Entity.js';
+
+export type AccessibleEntityProperty<T> = T extends Array<any>
+	? number
+	: T extends object
+	? keyof T
+	: never;
+
+export type DataFromInit<Init> = Init extends { [key: string]: any }
+	? {
+			[Key in keyof Init]: Init[Key];
+	  }
+	: Init extends Array<any>
+	? Init
+	: any;
+
+// reduces keys of an object to only ones with an optional
+// value
+export type DeletableKeys<T> = keyof {
+	[Key in keyof T as IfNullableThen<T[Key], Key>]: Key;
+};
+type IfNullableThen<T, Out> = undefined extends T
+	? Out
+	: null extends T
+	? Out
+	: never;
+
+export type EntityShape<E extends Entity<any, any>> = E extends Entity<
+	infer Value,
+	any
+>
+	? Value
+	: never;
+
+export type BaseEntityValue = { [Key: string]: any } | any[];
+
+export interface EntityChange {
+	oid: ObjectIdentifier;
+	isLocal: boolean;
+}
+
+export interface EntityChangeInfo {
+	isLocal?: boolean;
+}
+
+export type EntityEvents = {
+	change: (info: EntityChangeInfo) => void;
+	changeDeep: (
+		target: BaseEntity<any, any, any>,
+		info: EntityChangeInfo,
+	) => void;
+	delete: (info: EntityChangeInfo) => void;
+	restore: (info: EntityChangeInfo) => void;
+};
+
+export interface BaseEntity<
+	Init,
+	Value extends BaseEntityValue,
+	Snapshot = DataFromInit<Init>,
+> {
+	dispose: () => void;
+	subscribe<EventName extends keyof EntityEvents>(
+		event: EventName,
+		callback: EntityEvents[EventName],
+	): () => void;
+	get<Key extends keyof Value>(key: Key): Value[Key];
+	getAll(): Value;
+	getSnapshot(): Snapshot;
+	readonly deleted: boolean;
+	readonly updatedAt: number;
+	readonly uid: string;
+}
+
+export type DeepPartial<T> = {
+	[P in keyof T]?: T[P] extends Array<infer U>
+		? Array<DeepPartial<U>>
+		: T[P] extends ReadonlyArray<infer U>
+		? ReadonlyArray<DeepPartial<U>>
+		: DeepPartial<T[P]>;
+};
+
+export interface ObjectEntity<
+	Init,
+	Value extends BaseEntityValue,
+	Snapshot = DataFromInit<Init>,
+> extends BaseEntity<Init, Value, Snapshot> {
+	keys(): string[];
+	entries(): [string, Exclude<Value[keyof Value], undefined>][];
+	values(): Exclude<Value[keyof Value], undefined>[];
+	set<Key extends keyof Init>(key: Key, value: Init[Key]): void;
+	delete(key: DeletableKeys<Value>): void;
+	update(
+		value: DeepPartial<Init>,
+		options?: { replaceSubObjects?: boolean; merge?: boolean },
+	): void;
+	readonly isList: false;
+}
+
+export interface ListEntity<
+	Init,
+	Value extends BaseEntityValue,
+	Snapshot = DataFromInit<Init>,
+> extends Iterable<ListItemValue<Value>>,
+		BaseEntity<Init, Value, Snapshot> {
+	readonly isList: true;
+	readonly length: number;
+	push(value: ListItemInit<Init>): void;
+	insert(index: number, value: ListItemInit<Init>): void;
+	move(from: number, to: number): void;
+	moveItem(item: ListItemValue<Value>, to: number): void;
+	/**
+	 * A Set operation which adds a value if an equivalent value is not already present.
+	 * Object values are never the same.
+	 */
+	add(value: ListItemValue<Value>): void;
+	removeAll(item: ListItemValue<Value>): void;
+	removeFirst(item: ListItemValue<Value>): void;
+	removeLast(item: ListItemValue<Value>): void;
+	map<U>(callback: (value: ListItemValue<Value>, index: number) => U): U[];
+	filter(
+		callback: (value: ListItemValue<Value>, index: number) => boolean,
+	): ListItemValue<Value>[];
+	delete(index: number): void;
+	has(value: ListItemValue<Value>): boolean;
+	forEach(callback: (value: ListItemValue<Value>, index: number) => void): void;
+	some(predicate: (value: ListItemValue<Value>) => boolean): boolean;
+	every(predicate: (value: ListItemValue<Value>) => boolean): boolean;
+	find(
+		predicate: (value: ListItemValue<Value>) => boolean,
+	): ListItemValue<Value> | undefined;
+	includes(value: ListItemValue<Value>): boolean;
+}
+
+export type AnyEntity<
+	Init,
+	KeyValue extends BaseEntityValue,
+	Snapshot extends any,
+> =
+	| ListEntity<Init, KeyValue, Snapshot>
+	| ObjectEntity<Init, KeyValue, Snapshot>;
+
+export type ListItemValue<KeyValue> = KeyValue extends Array<infer T>
+	? T
+	: never;
+export type ListItemInit<Init> = Init extends Array<infer T> ? T : never;
+
+export type EntityDestructured<T extends AnyEntity<any, any, any> | null> =
+	| (T extends ListEntity<any, infer KeyValue, any>
+			? KeyValue
+			: T extends ObjectEntity<any, infer KeyValue, any>
+			? KeyValue
+			: never)
+	| (T extends null ? null : never);

package/src/files/EntityFile.ts

@@ -17,7 +17,6 @@ export class EntityFile extends EventSubscriber<EntityFileEvents> {
 	private _fileData: FileData | null = null;
 	private _loading = true;
 	private _failed = false;
-	private _disposed = false;
 	private _downloadRemote = false;
 
 	constructor(
@@ -39,7 +38,6 @@ export class EntityFile extends EventSubscriber<EntityFileEvents> {
 	[UPDATE] = (fileData: FileData) => {
 		this._loading = false;
 		this._failed = false;
-		this._disposed = false;
 		this._fileData = fileData;
 		if (fileData.file) {
 			if (this._objectUrl) {
@@ -78,10 +76,17 @@ export class EntityFile extends EventSubscriber<EntityFileEvents> {
 		return this._failed;
 	}
 
-	dispose = () => {
+	destroy = () => {
 		if (this._objectUrl) {
 			URL.revokeObjectURL(this._objectUrl);
 		}
-		this._disposed = true;
+		this.dispose();
 	};
+
+	getSnapshot() {
+		return {
+			id: this.id,
+			url: this.loading || this.failed ? undefined : this.url,
+		};
+	}
 }

package/src/files/FileManager.ts

@@ -108,11 +108,11 @@ export class FileManager {
 	 * Immediately returns an EntityFile to use, then either loads
 	 * the file from cache, local database, or the server.
 	 */
-	get = (id: string) => {
+	get = (id: string, options?: { downloadRemote?: boolean }) => {
 		if (this.files.has(id)) {
 			return this.files.get(id)!;
 		}
-		const file = new EntityFile(id);
+		const file = new EntityFile(id, options);
 		this.files.set(id, file);
 		this.load(file);
 		return file;
@@ -120,6 +120,7 @@ export class FileManager {
 
 	private load = async (file: EntityFile, retries = 0) => {
 		if (retries > 5) {
+			this.context.log('error', 'Failed to load file after 5 retries');
 			file[MARK_FAILED]();
 			return;
 		}
@@ -137,6 +138,7 @@ export class FileManager {
 						downloadRemote: file.downloadRemote,
 					});
 				} else {
+					this.context.log('error', 'Failed to load file', result);
 					file[MARK_FAILED]();
 					if (result.retry) {
 						// schedule a retry
@@ -183,7 +185,7 @@ export class FileManager {
 	};
 
 	private handleFileRefsDeleted = async (fileRefs: FileRef[]) => {
-		const tx = this.storage.createTransaction(['files'], 'readwrite');
+		const tx = this.storage.createTransaction(['files'], { mode: 'readwrite' });
 		await Promise.all(
 			fileRefs.map(async (fileRef) => {
 				try {

package/src/files/FileStorage.ts

@@ -53,8 +53,7 @@ export class FileStorage extends IDBService {
 					buffer,
 				} as StoredFileData);
 			},
-			'readwrite',
-			transaction,
+			{ mode: 'readwrite', transaction },
 		);
 	};
 
@@ -86,8 +85,7 @@ export class FileStorage extends IDBService {
 					remote: 'true',
 				} as StoredFileData);
 			},
-			'readwrite',
-			transaction,
+			{ mode: 'readwrite', transaction },
 		);
 	};
 
@@ -100,8 +98,7 @@ export class FileStorage extends IDBService {
 			(store) => {
 				return store.get(id);
 			},
-			'readonly',
-			transaction,
+			{ mode: 'readonly', transaction },
 		);
 		if (!raw) {
 			return undefined;
@@ -129,8 +126,7 @@ export class FileStorage extends IDBService {
 			(store) => {
 				return store.delete(id);
 			},
-			'readwrite',
-			transaction,
+			{ mode: 'readwrite', transaction },
 		);
 	}
 
@@ -152,8 +148,7 @@ export class FileStorage extends IDBService {
 					deletedAt: Date.now(),
 				} as StoredFileData);
 			},
-			'readwrite',
-			transaction,
+			{ mode: 'readwrite', transaction },
 		);
 	};
 
@@ -163,7 +158,7 @@ export class FileStorage extends IDBService {
 			(store) => {
 				return store.index('remote').getAll('false');
 			},
-			'readonly',
+			{ mode: 'readonly' },
 		);
 		return raw.map(this.hydrateFileData);
 	};
@@ -182,8 +177,7 @@ export class FileStorage extends IDBService {
 			(value, store) => {
 				iterator(this.hydrateFileData(value), store);
 			},
-			'readwrite',
-			transaction,
+			{ mode: 'readwrite', transaction },
 		);
 	};
 }

package/src/files/utils.ts

@@ -12,11 +12,15 @@ export function createFileData(file: File): FileData {
 	};
 }
 
-function isFile(value: any): value is File {
-	return value instanceof File;
+export function isFile(value: any): value is File {
+	return (
+		value instanceof File ||
+		(typeof Blob !== 'undefined' && value instanceof Blob)
+	);
 }
 
 /**
+ * MUTATES the value.
  * Replaces File values with refs and returns the normalized value.
  * The list of files passed to the second argument will be populated with the files found in the value.
  */

package/src/idb.ts

@@ -1,12 +1,21 @@
 import { roughSizeOfObject } from '@verdant-web/common';
 
+export function isAbortError(err: unknown) {
+	return err instanceof Error && err.name === 'AbortError';
+}
+
 export function storeRequestPromise<T>(request: IDBRequest<T>) {
 	return new Promise<T>((resolve, reject) => {
 		request.onsuccess = () => {
 			resolve(request.result);
 		};
 		request.onerror = () => {
-			reject(request.error);
+			if (request.error && isAbortError(request.error)) {
+				// TODO: is this the right thing to do?
+				resolve(request.result);
+			} else {
+				reject(request.error);
+			}
 		};
 	});
 }
@@ -29,7 +38,11 @@ export function cursorIterator<T>(
 			}
 		};
 		request.onerror = () => {
-			reject(request.error);
+			if (request.error && isAbortError(request.error)) {
+				resolve();
+			} else {
+				reject(request.error);
+			}
 		};
 	});
 }
@@ -53,7 +66,14 @@ export function getSizeOfObjectStore(
 			}
 		};
 		cursorReq.onerror = function (e) {
-			reject(e);
+			if (cursorReq.error && isAbortError(cursorReq.error)) {
+				resolve({
+					count: count,
+					size: size,
+				});
+			} else {
+				reject(cursorReq.error);
+			}
 		};
 		tx.oncomplete = function (e) {
 			resolve({
@@ -115,3 +135,31 @@ export async function getAllDatabaseNamesAndVersions(
 ) {
 	return indexedDB.databases();
 }
+
+export function createAbortableTransaction(
+	db: IDBDatabase,
+	storeNames: string[],
+	mode: 'readonly' | 'readwrite',
+	abortSignal?: AbortSignal,
+	log?: (...args: any[]) => void,
+) {
+	const tx = db.transaction(storeNames, mode);
+	if (abortSignal) {
+		const abort = () => {
+			log?.('debug', 'aborting transaction');
+			try {
+				tx.abort();
+			} catch (e) {
+				log?.('debug', 'aborting transaction failed', e);
+			}
+		};
+		abortSignal.addEventListener('abort', abort);
+		tx.addEventListener('error', () => {
+			abortSignal.removeEventListener('abort', abort);
+		});
+		tx.addEventListener('complete', () => {
+			abortSignal.removeEventListener('abort', abort);
+		});
+	}
+	return tx;
+}

package/src/index.ts

@@ -19,7 +19,7 @@ export type {
 	AccessibleEntityProperty,
 	AnyEntity,
 	EntityDestructured,
-} from './entities/Entity.js';
+} from './entities/types.js';
 export { ServerSync } from './sync/Sync.js';
 export type { SyncTransportMode } from './sync/Sync.js';
 export { EntityFile } from './files/EntityFile.js';

package/src/metadata/AckInfoStore.ts

@@ -24,7 +24,7 @@ export class AckInfoStore extends IDBService {
 		await this.run(
 			'info',
 			(store) => store.put({ type: 'ack', globalAckTimestamp: ack }),
-			'readwrite',
+			{ mode: 'readwrite' },
 		);
 	};
 }