@verdant-web/store 3.6.4 → 3.8.0

package/src/entities/Entity.ts

@@ -475,11 +475,15 @@ export class Entity<
 		}
 	};
 
+	private invalidateCachedView = () => {
+		this._viewData = undefined;
+		this.cachedView = undefined;
+	};
+
 	private change = (ev: EntityChange) => {
 		if (ev.oid === this.oid) {
 			// reset cached view
-			this._viewData = undefined;
-			this.cachedView = undefined;
+			this.invalidateCachedView();
 			if (!this.parent) {
 				this.changeRoot(ev);
 			} else {
@@ -751,15 +755,42 @@ export class Entity<
 		}
 	};
 
-	set = <Key extends keyof Init>(key: Key, value: Init[Key]) => {
+	set = (
+		key: any,
+		value: any,
+		options?: {
+			/**
+			 * Forces the creation of a change for this set even if the value is the same
+			 * as the current value for this key. This has an effect in situations where
+			 * offline changes from others are interleaved with local changes; when setting
+			 * a value to its current value (force=true), if that value were retroactively changed from
+			 * offline changes, the set would put it back to the value specified. If the
+			 * set is ignored because the value is the same (force=false), then retroactive
+			 * changes will be preserved.
+			 */
+			force: boolean;
+		},
+	) => {
 		assertNotSymbol(key);
-		this.addPendingOperations(
-			this.patchCreator.createSet(
-				this.oid,
-				key,
-				this.processInputValue(value, key),
-			),
-		);
+		if (!options?.force && this.get(key) === value) return;
+
+		if (this.isList) {
+			this.addPendingOperations(
+				this.patchCreator.createListSet(
+					this.oid,
+					key,
+					this.processInputValue(value, key),
+				),
+			);
+		} else {
+			this.addPendingOperations(
+				this.patchCreator.createSet(
+					this.oid,
+					key,
+					this.processInputValue(value, key),
+				),
+			);
+		}
 	};
 
 	/**
@@ -998,6 +1029,17 @@ export class Entity<
 		this.view.forEach(callback);
 	};
 
+	reduce = <U>(
+		callback: (
+			previousValue: U,
+			currentValue: ListItemValue<KeyValue>,
+			index: number,
+		) => U,
+		initialValue: U,
+	): U => {
+		return this.view.reduce(callback, initialValue);
+	};
+
 	some = (predicate: (value: ListItemValue<KeyValue>) => boolean): boolean => {
 		return this.view.some(predicate);
 	};
@@ -1034,6 +1076,11 @@ export class Entity<
 		return this.metadataFamily.get(oid).computeView(type === 'confirmed');
 	};
 	__getFamilyOids__ = () => this.metadataFamily.getAllOids();
+
+	__discardPendingOperation__ = (operation: Operation) => {
+		this.metadataFamily.discardPendingOperation(operation);
+		this.invalidateCachedView();
+	};
 }
 
 function assertNotSymbol<T>(key: T): asserts key is Exclude<T, symbol> {

package/src/entities/EntityMetadata.ts

@@ -176,19 +176,24 @@ export class EntityMetadata {
 			// FIXME: seems inefficient
 			// remove this incoming op from pending if it's in there
 			const pendingPrior = this.pendingOperations.length;
-			this.pendingOperations = this.pendingOperations.filter(
-				(pendingOp) => op.timestamp !== pendingOp.timestamp,
-			);
+			this.discardPendingOperation(op);
 			totalAdded -= pendingPrior - this.pendingOperations.length;
 		}
 		return totalAdded;
 	};
 
 	addPendingOperation = (operation: Operation) => {
+		// check to see if new operation supersedes the previous one
 		// we can assume pending ops are always newer
 		this.pendingOperations.push(operation);
 	};
 
+	discardPendingOperation = (operation: Operation) => {
+		this.pendingOperations = this.pendingOperations.filter(
+			(op) => op.timestamp !== operation.timestamp,
+		);
+	};
+
 	private applyOperations = (
 		base: any,
 		deleted: boolean,
@@ -361,4 +366,9 @@ export class EntityFamilyMetadata {
 		}
 		return Object.values(changes);
 	};
+
+	discardPendingOperation = (operation: Operation) => {
+		const ent = this.entities.get(operation.oid);
+		ent?.discardPendingOperation(operation);
+	};
 }

package/src/entities/EntityStore.ts

@@ -24,7 +24,6 @@ import { OperationBatcher } from './OperationBatcher.js';
 import { QueryableStorage } from '../queries/QueryableStorage.js';
 import { WeakEvent } from 'weak-event';
 import { processValueFiles } from '../files/utils.js';
-import { abort } from 'process';
 
 enum AbortReason {
 	Reset,
@@ -197,9 +196,6 @@ export class EntityStore extends Disposable {
 					});
 				});
 			} else {
-				if (this.cache.has(oid)) {
-					this.ctx.log('debug', 'Cache has', oid, ', an event should follow.');
-				}
 				event.invoke(this, {
 					oid,
 					baselines,
@@ -214,13 +210,12 @@ export class EntityStore extends Disposable {
 		};
 
 		// then, asynchronously add to the database
+		// this also emits messages to sync
+		// TODO: could messages be sent to sync before storage,
+		// so that realtime is lower latency? What would happen
+		// if the storage failed?
 		await this.meta.insertData(data, abortOptions);
 
-		// FIXME: entities hydrated here are not seeing
-		// the operations just inserted above!!
-		// IDEA: can we coordinate here with hydrate promises
-		// based on affected OIDs?
-
 		// recompute all affected documents for querying
 		const entities = await Promise.all(
 			allDocumentOids.map(async (oid) => {
@@ -459,6 +454,11 @@ export class EntityStore extends Disposable {
 		this.batcher.addOperations(operations);
 	};
 
+	discardPendingOperation = (operation: Operation) => {
+		const root = getOidRoot(operation.oid);
+		this.cache.get(root)?.deref()?.__discardPendingOperation__(operation);
+	};
+
 	/**
 	 * Loads initial Entity data from storage
 	 */

package/src/entities/OperationBatcher.ts

@@ -1,10 +1,14 @@
 import {
 	Batcher,
+	ObjectIdentifier,
 	Operation,
+	PropertyName,
 	generateId,
 	getOidRoot,
 	getUndoOperations,
 	groupPatchesByOid,
+	isSuperseded,
+	operationSupersedes,
 } from '@verdant-web/common';
 import { Metadata } from '../metadata/Metadata.js';
 import { Context } from '../context.js';
@@ -74,6 +78,40 @@ export class OperationBatcher {
 			'to storage / sync',
 		);
 		if (!operations.length) return;
+
+		// next block of logic computes superseding rules to eliminate
+		// operations which are 'overshadowed' by later ones on the same
+		// key.
+
+		const committed: Operation[] = [];
+		const supersessions: Record<
+			ObjectIdentifier,
+			Set<boolean | PropertyName>
+		> = {};
+		for (let i = operations.length - 1; i >= 0; i--) {
+			const op = operations[i];
+
+			// check for supersession from later operation which either
+			// covers the whole id (true) or this key
+			const existingSupersession = supersessions[op.oid];
+			if (existingSupersession && isSuperseded(op, existingSupersession)) {
+				this.entities.discardPendingOperation(op);
+				continue;
+			}
+
+			// determine if this operation supersedes others
+			const supersession = operationSupersedes(op);
+			if (supersession !== false) {
+				if (!supersessions[op.oid]) {
+					supersessions[op.oid] = new Set<boolean | PropertyName>();
+				}
+				supersessions[op.oid]!.add(supersession);
+			}
+
+			// add this operation to final list
+			committed.unshift(op);
+		}
+
 		// rewrite timestamps of all operations to now - this preserves
 		// the linear history of operations which are sent to the server.
 		// even if multiple batches are spun up in parallel and flushed
@@ -86,10 +124,14 @@ export class OperationBatcher {
 		// despite the provisional timestamp being earlier
 		// NOTE: this MUST be mutating the original operation object! this timestamp
 		// also serves as a unique ID for deduplication later.
-		for (const op of operations) {
+
+		// NOTE: need to rewind back in order to set timestamps correctly.
+		// cannot be done in reversed loop above or timestamps would be
+		// in reverse order.
+		for (const op of committed) {
 			op.timestamp = this.meta.now;
 		}
-		await this.commitOperations(operations, meta);
+		await this.commitOperations(committed, meta);
 	};
 
 	/**
@@ -148,9 +190,34 @@ export class OperationBatcher {
 		max = null,
 		timeout = this.defaultBatchTimeout,
 	}: {
+		/** Allows turning off undo for this batch, making it 'permanent' */
 		undoable?: boolean;
+		/**
+		 * Provide a stable name to any invocation of .batch() and the changes made
+		 * within run() will all be added to the same batch. If a batch hits the max
+		 * limit or timeout and is flushed, the name will be reused for a new batch
+		 * automatically. Provide a stable name to make changes from anywhere in your
+		 * app to be grouped together in the same batch with the same limit behavior.
+		 *
+		 * Limit configuration provided to each invocation of .batch() with the same
+		 * name will overwrite any other invocation's limit configuration. It's
+		 * recommended to provide limits in one place and only provide a name
+		 * in others.
+		 */
 		batchName?: string;
+		/**
+		 * The maximum number of operations the batch will hold before flushing
+		 * automatically. If null, the batch will not flush automatically based
+		 * on operation count.
+		 */
 		max?: number | null;
+		/**
+		 * The number of milliseconds to wait before flushing the batch automatically.
+		 * If null, the batch will not flush automatically based on time. It is not
+		 * recommended to set this to null, as an unflushed batch will never be written
+		 * to storage or sync. If you do require undefined timing in a batch, make sure
+		 * to always call .commit() on the batch yourself.
+		 */
 		timeout?: number | null;
 	} = {}): OperationBatch => {
 		const internalBatch = this.batcher.add({

package/src/entities/types.ts

@@ -87,7 +87,11 @@ export interface ObjectEntity<
 	readonly size: number;
 	entries(): [string, Exclude<Value[keyof Value], undefined>][];
 	values(): Exclude<Value[keyof Value], undefined>[];
-	set<Key extends keyof Init>(key: Key, value: Init[Key]): void;
+	set<Key extends keyof Init>(
+		key: Key,
+		value: Init[Key],
+		options?: { force?: boolean },
+	): void;
 	delete(key: DeletableKeys<Value>): void;
 	update(
 		value: DeepPartial<Init>,
@@ -136,6 +140,11 @@ export interface ListEntity<
 		BaseEntity<Init, Value, Snapshot> {
 	readonly isList: true;
 	readonly length: number;
+	set(
+		index: number,
+		value: ListItemInit<Init>,
+		options?: { force?: boolean },
+	): void;
 	push(value: ListItemInit<Init>): void;
 	insert(index: number, value: ListItemInit<Init>): void;
 	move(from: number, to: number): void;
@@ -149,6 +158,14 @@ export interface ListEntity<
 	removeFirst(item: ListItemValue<Value>): void;
 	removeLast(item: ListItemValue<Value>): void;
 	map<U>(callback: (value: ListItemValue<Value>, index: number) => U): U[];
+	reduce<U>(
+		callback: (
+			accumulator: U,
+			currentValue: ListItemValue<Value>,
+			index: number,
+		) => U,
+		initialValue: U,
+	): U;
 	filter(
 		callback: (value: ListItemValue<Value>, index: number) => boolean,
 	): ListItemValue<Value>[];

package/src/metadata/Metadata.ts

@@ -60,12 +60,16 @@ export class Metadata extends EventSubscriber<{
 
 	private context: Omit<Context, 'documentDb' | 'getNow'>;
 
+	private onOperation?: (operation: Operation) => void;
+
 	constructor({
 		disableRebasing,
 		context,
+		onOperation,
 	}: {
 		disableRebasing?: boolean;
 		context: Omit<Context, 'documentDb' | 'getNow'>;
+		onOperation?: (operation: Operation) => void;
 	}) {
 		super();
 		this.context = context;
@@ -76,6 +80,7 @@ export class Metadata extends EventSubscriber<{
 		this.ackInfo = new AckInfoStore(this.db);
 		this.messageCreator = new MessageCreator(this);
 		this.patchCreator = new PatchCreator(() => this.now);
+		this.onOperation = onOperation;
 		if (disableRebasing) this.disableRebasing = disableRebasing;
 	}
 
@@ -312,6 +317,10 @@ export class Metadata extends EventSubscriber<{
 
 		// we can now enqueue and check for rebase opportunities
 		this.tryAutonomousRebase();
+
+		if (this.onOperation) {
+			operations.forEach((o) => this.onOperation!(o));
+		}
 	};
 
 	/**
@@ -340,6 +349,10 @@ export class Metadata extends EventSubscriber<{
 
 		this.ack(operations[operations.length - 1].timestamp);
 
+		if (this.onOperation) {
+			operations.forEach((o) => this.onOperation!(o));
+		}
+
 		return affectedDocumentOids;
 	};
 

package/src/queries/BaseQuery.ts

@@ -42,6 +42,7 @@ export abstract class BaseQuery<T> extends Disposable {
 
 	readonly collection;
 	readonly key;
+	readonly isListQuery;
 
 	constructor({
 		initial,
@@ -53,6 +54,7 @@ export abstract class BaseQuery<T> extends Disposable {
 		super();
 		this._rawValue = initial;
 		this._value = initial;
+		this.isListQuery = Array.isArray(initial);
 		this._events = new EventSubscriber<BaseQueryEvents>(
 			(event: keyof BaseQueryEvents) => {
 				if (event === 'change') this._allUnsubscribedHandler?.(this);
@@ -98,6 +100,19 @@ export abstract class BaseQuery<T> extends Disposable {
 		return this._status;
 	}
 
+	private set status(v: QueryStatus) {
+		if (this._status === v) return;
+		this._status = v;
+		this._events.emit('statusChange', this._status);
+	}
+
+	get hasDeleted() {
+		if (this.isListQuery) {
+			return (this._rawValue as any[]).length !== (this._value as any[]).length;
+		}
+		return !!this._rawValue && !this._value;
+	}
+
 	/**
 	 * Subscribe to changes in the query value.
 	 *
@@ -138,16 +153,38 @@ export abstract class BaseQuery<T> extends Disposable {
 	protected setValue = (value: T) => {
 		this._rawValue = value;
 		this.subscribeToDeleteAndRestore(this._rawValue);
-		this._value = filterResultSet(value);
-		// validate the value
-		if (
-			Array.isArray(this._value) &&
-			this._value.some((v) => v.getSnapshot() === null)
-		) {
-			debugger;
+		const filtered = filterResultSet(value);
+
+		// prevent excess change notifications by diffing
+		// value by identity for single-value queries,
+		// and by item identity for multi-value
+		let changed = true;
+		// always fire change when going from initial to ready
+		if (this.status === 'initializing' || this.status === 'initial') {
+			changed = true;
+		} else {
+			// compare values by identity, after filtering.
+			if (this.isListQuery) {
+				if (
+					(this._value as any[]).length === (filtered as any[]).length &&
+					(this._value as any[]).every((v, i) => v === (filtered as any[])[i])
+				) {
+					changed = false;
+				}
+			} else {
+				if (this._value === filtered) {
+					changed = false;
+				}
+			}
+		}
+
+		this._value = filtered;
+
+		if (changed) {
+			this.context.log('debug', 'Query value changed', this.key);
+			this._events.emit('change', this._value);
 		}
-		this._status = 'ready';
-		this._events.emit('change', this._value);
+		this.status = 'ready';
 	};
 
 	// re-applies filtering if results have changed
@@ -186,10 +223,10 @@ export abstract class BaseQuery<T> extends Disposable {
 	execute = () => {
 		this.context.log('debug', 'Executing query', this.key);
 
-		if (this._status === 'initial') {
-			this._status = 'initializing';
-		} else if (this._status === 'ready') {
-			this._status = 'revalidating';
+		if (this.status === 'initial') {
+			this.status = 'initializing';
+		} else if (this.status === 'ready') {
+			this.status = 'revalidating';
 		}
 		// no status change needed if already in a 'running' status.
 

package/src/sync/Sync.ts

@@ -173,6 +173,11 @@ export interface ServerSyncOptions<Profile = any, Presence = any>
 	 * but is not yet thoroughly vetted.
 	 */
 	useBroadcastChannel?: boolean;
+	/**
+	 * Listen for outgoing messages from the client to the server.
+	 * Not sure why you want to do this, but be careful.
+	 */
+	onOutgoingMessage?: (message: ClientMessage) => void;
 }
 
 export class ServerSync<Presence = any, Profile = any>
@@ -196,6 +201,8 @@ export class ServerSync<Presence = any, Profile = any>
 
 	readonly presence: PresenceManager<Profile, Presence>;
 
+	private onOutgoingMessage?: (message: ClientMessage) => void;
+
 	private log;
 
 	constructor(
@@ -211,6 +218,7 @@ export class ServerSync<Presence = any, Profile = any>
 			presenceUpdateBatchTimeout,
 			defaultProfile,
 			useBroadcastChannel,
+			onOutgoingMessage,
 		}: ServerSyncOptions<Profile, Presence>,
 		{
 			meta,
@@ -230,6 +238,7 @@ export class ServerSync<Presence = any, Profile = any>
 		this.meta = meta;
 		this.onData = onData;
 		this.log = ctx.log;
+		this.onOutgoingMessage = onOutgoingMessage;
 		this.presence = new PresenceManager({
 			initialPresence,
 			defaultProfile,
@@ -444,9 +453,10 @@ export class ServerSync<Presence = any, Profile = any>
 		return this.pushPullSync.interval;
 	}
 
-	send = (message: ClientMessage) => {
+	send = async (message: ClientMessage) => {
 		if (this.activeSync.status === 'active') {
-			return this.activeSync.send(message);
+			await this.activeSync.send(message);
+			this.onOutgoingMessage?.(message);
 		}
 	};