@verdant-web/store 3.6.3 → 3.7.0

package/src/__tests__/fixtures/testStorage.ts

@@ -1,8 +1,14 @@
-import { createMigration, schema } from '@verdant-web/common';
+import {
+	ClientMessage,
+	createMigration,
+	Operation,
+	schema,
+} from '@verdant-web/common';
 // @ts-ignore
 import { IDBFactory } from 'fake-indexeddb';
 import { ClientWithCollections, ClientDescriptor } from '../../index.js';
 import { METADATA_VERSION_KEY } from '../../client/constants.js';
+import { Sync } from '../../sync/Sync.js';
 
 export const todoCollection = schema.collection({
 	name: 'todo',
@@ -91,11 +97,13 @@ export function createTestStorage({
 	disableRebasing = false,
 	metadataVersion,
 	log,
+	onOperation,
 }: {
 	idb?: IDBFactory;
 	disableRebasing?: boolean;
 	metadataVersion?: number;
 	log?: (...args: any[]) => void;
+	onOperation?: (op: Operation) => void;
 } = {}) {
 	const storage = new ClientDescriptor({
 		schema: testSchema,
@@ -105,6 +113,7 @@ export function createTestStorage({
 		disableRebasing,
 		log,
 		[METADATA_VERSION_KEY]: metadataVersion,
+		onOperation,
 	}).open();
 	return storage as Promise<ClientWithCollections>;
 }

package/src/__tests__/mutations.test.ts

@@ -54,4 +54,57 @@ describe('mutations', () => {
 		expect(itemAExists).toBeNull();
 		expect(itemBExists === itemB).toBe(true);
 	});
+
+	describe('on entities', () => {
+		it('should allow them to delete themselves (root level)', async () => {
+			const client = await createTestStorage();
+
+			const itemA = await client.todos.put({
+				id: '1',
+				content: 'itemA',
+				category: 'test',
+			});
+
+			await itemA.deleteSelf();
+
+			const itemAExists = await client.todos.get('1').resolved;
+
+			expect(itemAExists).toBeNull();
+		});
+
+		it('should allow them to delete themselves (nested array)', async () => {
+			const client = await createTestStorage();
+
+			const itemA = await client.todos.put({
+				id: '1',
+				content: 'itemA',
+				category: 'test',
+				attachments: [
+					{
+						name: 'foo',
+					},
+				],
+			});
+
+			itemA.get('attachments').get(0).deleteSelf();
+
+			expect(itemA.get('attachments').length).toBe(0);
+		});
+
+		it('should allow them to delete themselves (nested map)', async () => {
+			const client = await createTestStorage();
+
+			const weird = await client.weirds.put({
+				objectMap: {
+					foo: {
+						content: 'bar',
+					},
+				},
+			});
+
+			weird.get('objectMap').get('foo').deleteSelf();
+
+			expect(weird.get('objectMap').size).toBe(0);
+		});
+	});
 });

package/src/client/Client.ts

@@ -197,7 +197,7 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		return this.entities.batch;
 	}
 
-	stats = async () => {
+	stats = async (): Promise<ClientStats> => {
 		const collectionNames = Object.keys(this.schema.collections);
 		let collections = {} as Record<string, { count: number; size: number }>;
 		if (this.disposed) {
@@ -427,3 +427,16 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		await this.import(exportData);
 	};
 }
+
+export interface ClientStats {
+	collections: Record<string, { count: number; size: number }>;
+	meta: {
+		baselinesSize: { count: number; size: number };
+		operationsSize: { count: number; size: number };
+	};
+	storage: StorageEstimate | undefined;
+	totalMetaSize: number;
+	totalCollectionsSize: number;
+	metaToDataRatio: number;
+	quotaUsage: number | undefined;
+}

package/src/client/ClientDescriptor.ts

@@ -1,6 +1,7 @@
 import {
 	EventSubscriber,
 	Migration,
+	Operation,
 	StorageSchema,
 	hashObject,
 } from '@verdant-web/common';
@@ -60,6 +61,13 @@ export interface ClientDescriptorOptions<Presence = any, Profile = any> {
 	 * Configuration for file management
 	 */
 	files?: FileManagerConfig;
+
+	/**
+	 * Listen for operations as they are applied to the database.
+	 * Wouldn't recommend using this unless you know what you're doing.
+	 * It's a very hot code path...
+	 */
+	onOperation?: (operation: Operation) => void;
 	/**
 	 * Enables experimental WeakRef usage to cull documents
 	 * from cache that aren't being used. This is a performance
@@ -173,6 +181,7 @@ export class ClientDescriptor<
 		const meta = new Metadata({
 			context,
 			disableRebasing: init.disableRebasing,
+			onOperation: init.onOperation,
 		});
 
 		// verify schema integrity

package/src/entities/Entity.test.ts

@@ -51,7 +51,9 @@ describe('Entity', () => {
 		},
 	} as const;
 
-	function createTestEntity() {
+	function createTestEntity({
+		onPendingOperations = vi.fn(),
+	}: { onPendingOperations?: () => void } = {}) {
 		const events: EntityStoreEvents = {
 			add: new WeakEvent(),
 			replace: new WeakEvent(),
@@ -70,7 +72,7 @@ describe('Entity', () => {
 			events,
 			metadataFamily: new EntityFamilyMetadata({
 				ctx: mockContext,
-				onPendingOperations: vi.fn(),
+				onPendingOperations,
 				rootOid: 'test/1',
 			}),
 			files: {
@@ -79,6 +81,7 @@ describe('Entity', () => {
 			} as any as FileManager,
 			patchCreator,
 			readonlyKeys: ['id'],
+			deleteSelf: vi.fn(),
 		});
 
 		function initialize(data: any) {
@@ -238,4 +241,30 @@ describe('Entity', () => {
 			expect(entity.get('string')).toBe('new world');
 		});
 	});
+
+	describe('dropping pending operations', () => {
+		it('drops the correct operation', () => {
+			const onPendingOperations = vi.fn();
+			const { entity, initialize } = createTestEntity({ onPendingOperations });
+
+			initialize({
+				id: 'hi',
+				string: 'world',
+				number: 1,
+				nullable: null,
+				nonNullable: { first: { inner: 'foo' } },
+				map: {},
+				list: [],
+			});
+
+			entity.update({ string: 'new world' });
+			expect(onPendingOperations).toHaveBeenCalledTimes(1);
+			const operation = onPendingOperations.mock.calls[0][0][0];
+			console.log(operation);
+
+			entity.__discardPendingOperation__(operation);
+
+			expect(entity.get('string')).toBe('world');
+		});
+	});
 });

package/src/entities/Entity.ts

@@ -58,6 +58,7 @@ export interface EntityInit {
 	fieldPath?: (string | number)[];
 	patchCreator: PatchCreator;
 	events: EntityStoreEvents;
+	deleteSelf: () => void;
 }
 
 export class Entity<
@@ -94,6 +95,9 @@ export class Entity<
 	// only used for root entities to track delete/restore state.
 	private wasDeletedLastChange = false;
 	private cachedView: any | undefined = undefined;
+	// provided from external creators, this is a method to delete
+	// this entity.
+	private _deleteSelf: () => void;
 
 	constructor({
 		oid,
@@ -106,6 +110,7 @@ export class Entity<
 		files,
 		patchCreator,
 		events,
+		deleteSelf,
 	}: EntityInit) {
 		super();
 
@@ -125,6 +130,7 @@ export class Entity<
 		this.metadataFamily = metadataFamily;
 		this.events = events;
 		this.parent = parent;
+		this._deleteSelf = deleteSelf;
 
 		// TODO: should any but the root entity be listening to these?
 		if (!this.parent) {
@@ -366,6 +372,15 @@ export class Entity<
 		return this.viewData.fromOlderVersion;
 	}
 
+	/**
+	 * Returns the storage namespace this entity came from. For example, if you
+	 * have multiple stores initialized from the same schema, you can use this
+	 * to figure out where an isolated entity was created / stored.
+	 */
+	get namespace() {
+		return this.ctx.namespace;
+	}
+
 	/**
 	 * Pruning - when entities have invalid children, we 'prune' that
 	 * data up to the nearest prunable point - a nullable field,
@@ -460,30 +475,19 @@ export class Entity<
 		}
 	};
 
+	private invalidateCachedView = () => {
+		this._viewData = undefined;
+		this.cachedView = undefined;
+	};
+
 	private change = (ev: EntityChange) => {
 		if (ev.oid === this.oid) {
 			// reset cached view
-			this._viewData = undefined;
-			this.cachedView = undefined;
-			// chain deepChanges to parents
-			this.deepChange(this, ev);
-			// emit the change, it's for us
-			this.ctx.log('Emitting change event', this.oid);
-			this.emit('change', { isLocal: ev.isLocal });
-			// for root entities, we need to go ahead and decide if we're
-			// deleted or not - so queries can exclude us if we are.
+			this.invalidateCachedView();
 			if (!this.parent) {
-				// newly deleted - emit event
-				if (this.deleted && !this.wasDeletedLastChange) {
-					this.ctx.log('debug', 'Entity deleted', this.oid);
-					this.emit('delete', { isLocal: ev.isLocal });
-					this.wasDeletedLastChange = true;
-				} else if (!this.deleted && this.wasDeletedLastChange) {
-					this.ctx.log('debug', 'Entity restored', this.oid);
-					// newly restored - emit event
-					this.emit('restore', { isLocal: ev.isLocal });
-					this.wasDeletedLastChange = false;
-				}
+				this.changeRoot(ev);
+			} else {
+				this.changeNested(ev);
 			}
 		} else {
 			// forward it to the correct family member. if none exists
@@ -494,6 +498,36 @@ export class Entity<
 			}
 		}
 	};
+	private changeRoot = (ev: EntityChange) => {
+		// for root entities, we need to determine if we're deleted or not
+		// before firing any events
+		if (this.deleted) {
+			if (!this.wasDeletedLastChange) {
+				this.ctx.log('debug', 'Entity deleted', this.oid);
+				this.emit('delete', { isLocal: ev.isLocal });
+				this.wasDeletedLastChange = true;
+			}
+			// already deleted, do nothing.
+		} else {
+			if (this.wasDeletedLastChange) {
+				this.ctx.log('debug', 'Entity restored', this.oid);
+				this.emit('restore', { isLocal: ev.isLocal });
+				this.wasDeletedLastChange = false;
+			}
+			// emit deepchange, too
+			this.deepChange(this, ev);
+			// emit the change, it's for us
+			this.ctx.log('Emitting change event', this.oid);
+			this.emit('change', { isLocal: ev.isLocal });
+		}
+	};
+	private changeNested = (ev: EntityChange) => {
+		// chain deepChanges to parents
+		this.deepChange(this, ev);
+		// emit the change, it's for us
+		this.ctx.log('Emitting change event', this.oid);
+		this.emit('change', { isLocal: ev.isLocal });
+	};
 	protected deepChange = (target: Entity, ev: EntityChange) => {
 		// reset cached deep updated at timestamp; either this
 		// entity or children have changed
@@ -530,6 +564,7 @@ export class Entity<
 			fieldPath: [...this.fieldPath, key],
 			patchCreator: this.patchCreator,
 			events: this.events,
+			deleteSelf: this.delete.bind(this, key),
 		});
 	};
 
@@ -720,15 +755,42 @@ export class Entity<
 		}
 	};
 
-	set = <Key extends keyof Init>(key: Key, value: Init[Key]) => {
+	set = (
+		key: any,
+		value: any,
+		options?: {
+			/**
+			 * Forces the creation of a change for this set even if the value is the same
+			 * as the current value for this key. This has an effect in situations where
+			 * offline changes from others are interleaved with local changes; when setting
+			 * a value to its current value (force=true), if that value were retroactively changed from
+			 * offline changes, the set would put it back to the value specified. If the
+			 * set is ignored because the value is the same (force=false), then retroactive
+			 * changes will be preserved.
+			 */
+			force: boolean;
+		},
+	) => {
 		assertNotSymbol(key);
-		this.addPendingOperations(
-			this.patchCreator.createSet(
-				this.oid,
-				key,
-				this.processInputValue(value, key),
-			),
-		);
+		if (!options?.force && this.get(key) === value) return;
+
+		if (this.isList) {
+			this.addPendingOperations(
+				this.patchCreator.createListSet(
+					this.oid,
+					key,
+					this.processInputValue(value, key),
+				),
+			);
+		} else {
+			this.addPendingOperations(
+				this.patchCreator.createSet(
+					this.oid,
+					key,
+					this.processInputValue(value, key),
+				),
+			);
+		}
 	};
 
 	/**
@@ -781,6 +843,13 @@ export class Entity<
 		return Object.values(this.view);
 	};
 
+	get size() {
+		if (this.isList) {
+			return this.length;
+		}
+		return this.keys().length;
+	}
+
 	update = (
 		data: any,
 		{
@@ -960,6 +1029,17 @@ export class Entity<
 		this.view.forEach(callback);
 	};
 
+	reduce = <U>(
+		callback: (
+			previousValue: U,
+			currentValue: ListItemValue<KeyValue>,
+			index: number,
+		) => U,
+		initialValue: U,
+	): U => {
+		return this.view.reduce(callback, initialValue);
+	};
+
 	some = (predicate: (value: ListItemValue<KeyValue>) => boolean): boolean => {
 		return this.view.some(predicate);
 	};
@@ -976,11 +1056,31 @@ export class Entity<
 
 	includes = this.has;
 
+	/**
+	 * Deletes this entity. WARNING: this can be tricky to
+	 * use correctly. You must not reference this entity
+	 * instance in any way after the deletion happens, or
+	 * you will get an error!
+	 *
+	 * It's a little easier to delete using client.delete
+	 * if you can manage it with your app's code. For example,
+	 * in React, use hooks.useClient() to get the client and
+	 * call delete from there.
+	 */
+	deleteSelf = () => {
+		return this._deleteSelf();
+	};
+
 	// TODO: make these escape hatches unnecessary
 	__getViewData__ = (oid: ObjectIdentifier, type: 'confirmed' | 'pending') => {
 		return this.metadataFamily.get(oid).computeView(type === 'confirmed');
 	};
 	__getFamilyOids__ = () => this.metadataFamily.getAllOids();
+
+	__discardPendingOperation__ = (operation: Operation) => {
+		this.metadataFamily.discardPendingOperation(operation);
+		this.invalidateCachedView();
+	};
 }
 
 function assertNotSymbol<T>(key: T): asserts key is Exclude<T, symbol> {

package/src/entities/EntityMetadata.ts

@@ -176,19 +176,24 @@ export class EntityMetadata {
 			// FIXME: seems inefficient
 			// remove this incoming op from pending if it's in there
 			const pendingPrior = this.pendingOperations.length;
-			this.pendingOperations = this.pendingOperations.filter(
-				(pendingOp) => op.timestamp !== pendingOp.timestamp,
-			);
+			this.discardPendingOperation(op);
 			totalAdded -= pendingPrior - this.pendingOperations.length;
 		}
 		return totalAdded;
 	};
 
 	addPendingOperation = (operation: Operation) => {
+		// check to see if new operation supersedes the previous one
 		// we can assume pending ops are always newer
 		this.pendingOperations.push(operation);
 	};
 
+	discardPendingOperation = (operation: Operation) => {
+		this.pendingOperations = this.pendingOperations.filter(
+			(op) => op.timestamp !== operation.timestamp,
+		);
+	};
+
 	private applyOperations = (
 		base: any,
 		deleted: boolean,
@@ -361,4 +366,9 @@ export class EntityFamilyMetadata {
 		}
 		return Object.values(changes);
 	};
+
+	discardPendingOperation = (operation: Operation) => {
+		const ent = this.entities.get(operation.oid);
+		ent?.discardPendingOperation(operation);
+	};
 }

package/src/entities/EntityStore.ts

@@ -24,7 +24,6 @@ import { OperationBatcher } from './OperationBatcher.js';
 import { QueryableStorage } from '../queries/QueryableStorage.js';
 import { WeakEvent } from 'weak-event';
 import { processValueFiles } from '../files/utils.js';
-import { abort } from 'process';
 
 enum AbortReason {
 	Reset,
@@ -197,9 +196,6 @@ export class EntityStore extends Disposable {
 					});
 				});
 			} else {
-				if (this.cache.has(oid)) {
-					this.ctx.log('debug', 'Cache has', oid, ', an event should follow.');
-				}
 				event.invoke(this, {
 					oid,
 					baselines,
@@ -214,13 +210,12 @@ export class EntityStore extends Disposable {
 		};
 
 		// then, asynchronously add to the database
+		// this also emits messages to sync
+		// TODO: could messages be sent to sync before storage,
+		// so that realtime is lower latency? What would happen
+		// if the storage failed?
 		await this.meta.insertData(data, abortOptions);
 
-		// FIXME: entities hydrated here are not seeing
-		// the operations just inserted above!!
-		// IDEA: can we coordinate here with hydrate promises
-		// based on affected OIDs?
-
 		// recompute all affected documents for querying
 		const entities = await Promise.all(
 			allDocumentOids.map(async (oid) => {
@@ -451,6 +446,7 @@ export class EntityStore extends Disposable {
 			metadataFamily: metadataFamily,
 			patchCreator: this.meta.patchCreator,
 			events: this.events,
+			deleteSelf: this.delete.bind(this, oid),
 		});
 	};
 
@@ -458,6 +454,11 @@ export class EntityStore extends Disposable {
 		this.batcher.addOperations(operations);
 	};
 
+	discardPendingOperation = (operation: Operation) => {
+		const root = getOidRoot(operation.oid);
+		this.cache.get(root)?.deref()?.__discardPendingOperation__(operation);
+	};
+
 	/**
 	 * Loads initial Entity data from storage
 	 */

package/src/entities/OperationBatcher.ts

@@ -1,10 +1,14 @@
 import {
 	Batcher,
+	ObjectIdentifier,
 	Operation,
+	PropertyName,
 	generateId,
 	getOidRoot,
 	getUndoOperations,
 	groupPatchesByOid,
+	isSuperseded,
+	operationSupersedes,
 } from '@verdant-web/common';
 import { Metadata } from '../metadata/Metadata.js';
 import { Context } from '../context.js';
@@ -74,6 +78,40 @@ export class OperationBatcher {
 			'to storage / sync',
 		);
 		if (!operations.length) return;
+
+		// next block of logic computes superseding rules to eliminate
+		// operations which are 'overshadowed' by later ones on the same
+		// key.
+
+		const committed: Operation[] = [];
+		const supersessions: Record<
+			ObjectIdentifier,
+			Set<boolean | PropertyName>
+		> = {};
+		for (let i = operations.length - 1; i >= 0; i--) {
+			const op = operations[i];
+
+			// check for supersession from later operation which either
+			// covers the whole id (true) or this key
+			const existingSupersession = supersessions[op.oid];
+			if (existingSupersession && isSuperseded(op, existingSupersession)) {
+				this.entities.discardPendingOperation(op);
+				continue;
+			}
+
+			// determine if this operation supersedes others
+			const supersession = operationSupersedes(op);
+			if (supersession !== false) {
+				if (!supersessions[op.oid]) {
+					supersessions[op.oid] = new Set<boolean | PropertyName>();
+				}
+				supersessions[op.oid]!.add(supersession);
+			}
+
+			// add this operation to final list
+			committed.unshift(op);
+		}
+
 		// rewrite timestamps of all operations to now - this preserves
 		// the linear history of operations which are sent to the server.
 		// even if multiple batches are spun up in parallel and flushed
@@ -86,10 +124,14 @@ export class OperationBatcher {
 		// despite the provisional timestamp being earlier
 		// NOTE: this MUST be mutating the original operation object! this timestamp
 		// also serves as a unique ID for deduplication later.
-		for (const op of operations) {
+
+		// NOTE: need to rewind back in order to set timestamps correctly.
+		// cannot be done in reversed loop above or timestamps would be
+		// in reverse order.
+		for (const op of committed) {
 			op.timestamp = this.meta.now;
 		}
-		await this.commitOperations(operations, meta);
+		await this.commitOperations(committed, meta);
 	};
 
 	/**
@@ -148,9 +190,34 @@ export class OperationBatcher {
 		max = null,
 		timeout = this.defaultBatchTimeout,
 	}: {
+		/** Allows turning off undo for this batch, making it 'permanent' */
 		undoable?: boolean;
+		/**
+		 * Provide a stable name to any invocation of .batch() and the changes made
+		 * within run() will all be added to the same batch. If a batch hits the max
+		 * limit or timeout and is flushed, the name will be reused for a new batch
+		 * automatically. Provide a stable name to make changes from anywhere in your
+		 * app to be grouped together in the same batch with the same limit behavior.
+		 *
+		 * Limit configuration provided to each invocation of .batch() with the same
+		 * name will overwrite any other invocation's limit configuration. It's
+		 * recommended to provide limits in one place and only provide a name
+		 * in others.
+		 */
 		batchName?: string;
+		/**
+		 * The maximum number of operations the batch will hold before flushing
+		 * automatically. If null, the batch will not flush automatically based
+		 * on operation count.
+		 */
 		max?: number | null;
+		/**
+		 * The number of milliseconds to wait before flushing the batch automatically.
+		 * If null, the batch will not flush automatically based on time. It is not
+		 * recommended to set this to null, as an unflushed batch will never be written
+		 * to storage or sync. If you do require undefined timing in a batch, make sure
+		 * to always call .commit() on the batch yourself.
+		 */
 		timeout?: number | null;
 	} = {}): OperationBatch => {
 		const internalBatch = this.batcher.add({