@verdant-web/store 4.2.0 → 4.3.0

package/src/entities/Entity.ts

@@ -17,6 +17,7 @@ import {
 	isFileRef,
 	isNullable,
 	isObject,
+	isObjectRef,
 	isRef,
 	maybeGetOid,
 	memoByKeys,
@@ -297,7 +298,12 @@ export class Entity<
 						break;
 					}
 				} else {
-					this.cachedView[key] = child;
+					// special case - rewrite undefined fields to null
+					if (isNullable(schema) && child === undefined) {
+						this.cachedView[key] = null;
+					} else {
+						this.cachedView[key] = child;
+					}
 				}
 			}
 		}
@@ -325,6 +331,33 @@ export class Entity<
 		return !!this.validate();
 	}
 
+	/**
+	 * Returns true if this or any child is invalid (pruned)
+	 */
+	get deepInvalid(): boolean {
+		if (this.invalid) return true;
+		if (Array.isArray(this.rawView)) {
+			for (let i = 0; i < this.rawView.length; i++) {
+				if (isObjectRef(this.rawView[i])) {
+					const child = this.getChild(i, this.rawView[i].id);
+					if (child.deepInvalid) {
+						return true;
+					}
+				}
+			}
+		} else if (isObject(this.rawView)) {
+			for (const key in this.rawView) {
+				if (isObjectRef(this.rawView[key])) {
+					const child = this.getChild(key, this.rawView[key].id);
+					if (child.deepInvalid) {
+						return true;
+					}
+				}
+			}
+		}
+		return false;
+	}
+
 	get isList() {
 		// have to turn TS off here as our two interfaces both implement
 		// const values for this boolean.
@@ -418,7 +451,7 @@ export class Entity<
 					field: this.schema,
 					value: this.rawView,
 					fieldPath: this.fieldPath,
-					depth: 1,
+					expectRefs: true,
 				}) ?? undefined;
 			return this.validationError;
 		},
@@ -456,6 +489,37 @@ export class Entity<
 		}
 	};
 
+	private rawViewWithMappedChildren = (
+		mapper: (child: Entity | EntityFile) => any,
+	) => {
+		const view = this.rawView;
+		if (!view) {
+			return null;
+		}
+		if (Array.isArray(view)) {
+			const mapped = view.map((value, i) => {
+				if (isRef(value)) {
+					return mapper(this.getChild(i, value.id));
+				} else {
+					return value;
+				}
+			});
+			assignOid(mapped, this.oid);
+			return mapped;
+		} else {
+			const mapped = Object.entries(view).reduce((acc, [key, value]) => {
+				if (isRef(value)) {
+					acc[key as any] = mapper(this.getChild(key, value.id));
+				} else {
+					acc[key as any] = value;
+				}
+				return acc;
+			}, {} as any);
+			assignOid(mapped, this.oid);
+			return mapped;
+		}
+	};
+
 	/**
 	 * A current snapshot of this Entity's data, including nested
 	 * Entities.
@@ -464,10 +528,62 @@ export class Entity<
 		return this.viewWithMappedChildren((child) => child.getSnapshot());
 	};
 
+	/**
+	 * A snapshot of this Entity with unpruned (invalid) data. This will
+	 * not conform to the entity schema and should be used carefully.
+	 *
+	 * Can be used to inspect or recover invalid, pruned data not
+	 * otherwise accessible.
+	 */
+	getUnprunedSnapshot = (): any => {
+		return this.rawViewWithMappedChildren((child) => {
+			if (child instanceof EntityFile) return child.getSnapshot();
+			return child.getUnprunedSnapshot();
+		});
+	};
+
 	// change management methods (internal use only)
 	private addPendingOperations = (operations: Operation[]) => {
-		this.ctx.log('debug', 'Entity: adding pending operations', this.oid);
+		this.ctx.log(
+			'debug',
+			'Entity: adding pending operations',
+			this.oid,
+			operations,
+		);
+
+		// special case -- if this entity is pruned, any changes we apply to it
+		// will be in relation to 'exposed' pruned data, not the 'real world'
+		// data that's backing it. That means those changes will produce unexpected
+		// or further invalid results. To avoid this, we basically stamp in the
+		// pruned version of this entity before proceeding.
+		//
+		// as an example of a failure mode without this check, consider a list:
+		// [1, 2, <pruned>, 4, 5]
+		// the user sees: [1, 2, 4, 5]
+		// when they try to replace the item at index 2 with "0" (they see "4"), they
+		// actually replace the invisible pruned item, resulting in [1, 2, 0, 4, 5]
+		// being the result when they expected [1, 2, 0, 5].
+		//
+		// To "stamp" the data before applying user changes, we diff the snapshot
+		// (which is the pruned version) with the current state of the entity.
+		if (this.deepInvalid) {
+			this.ctx.log(
+				'warn',
+				'Changes are being applied to a pruned entity. This means that the pruned version is being treated as the new baseline and any pruned invalid data is lost.',
+				this.oid,
+			);
+			this.canonizePrunedVersion();
+		}
 
+		this.applyPendingOperations(operations);
+	};
+
+	// naming is fuzzy here, but this method was split out from
+	// addPendingOperations since that method also conditionally canonizes
+	// the pruned snapshot, and I wanted to keep the actual insertion of
+	// the ops in one place, so leaving it as part of addPendingOperations
+	// would introduce infinite recursion when canonizing.
+	private applyPendingOperations = (operations: Operation[]) => {
 		// apply authz to all operations
 		if (this.access) {
 			for (const op of operations) {
@@ -481,6 +597,21 @@ export class Entity<
 		}
 	};
 
+	private canonizePrunedVersion = () => {
+		const snapshot = this.getSnapshot();
+		const unprunedSnapshot = this.getUnprunedSnapshot();
+		const operations = this.patchCreator.createDiff(
+			unprunedSnapshot,
+			snapshot,
+			{
+				authz: this.access,
+				merge: false,
+			},
+		);
+
+		this.applyPendingOperations(operations);
+	};
+
 	private addConfirmedData = (data: EntityStoreEventData) => {
 		this.ctx.log('debug', 'Entity: adding confirmed data', this.oid);
 		const changes = this.metadataFamily.addConfirmedData(data);
@@ -927,7 +1058,7 @@ export class Entity<
 		this.addPendingOperations(
 			this.patchCreator.createDiff(this.getSnapshot(), changes, {
 				mergeUnknownObjects: !replaceSubObjects,
-				defaultUndefined: merge,
+				merge,
 			}),
 		);
 	};

package/src/entities/EntityMetadata.ts

@@ -1,4 +1,5 @@
 import {
+	AuthorizationKey,
 	DocumentBaseline,
 	ObjectIdentifier,
 	Operation,
@@ -20,7 +21,7 @@ export type EntityMetadataView = {
 	empty: boolean;
 	updatedAt: number;
 	latestTimestamp: string | null;
-	authz?: string;
+	authz?: AuthorizationKey;
 };
 
 export class EntityMetadata {
@@ -101,7 +102,7 @@ export class EntityMetadata {
 					// we don't use after for pending ops, they're all
 					// logically in the future
 					null,
-			  );
+				);
 		if (pendingResult.authz) {
 			authz = pendingResult.authz;
 		}
@@ -221,10 +222,10 @@ export class EntityMetadata {
 		latestTimestamp: string | null;
 		deleted: boolean;
 		futureSeen: string | undefined;
-		authz?: string;
+		authz?: AuthorizationKey;
 	} => {
 		let futureSeen: string | undefined = undefined;
-		let authz: string | undefined = undefined;
+		let authz: AuthorizationKey | undefined = undefined;
 
 		const now = this.ctx.time.now;
 		for (const op of operations) {

package/src/entities/EntityStore.ts

@@ -1,4 +1,5 @@
 import {
+	AuthorizationKey,
 	DocumentBaseline,
 	ObjectIdentifier,
 	Operation,
@@ -12,16 +13,15 @@ import {
 	groupPatchesByRootOid,
 	isRootOid,
 	removeOidsFromAllSubObjects,
-	AuthorizationKey,
 } from '@verdant-web/common';
+import { WeakEvent } from 'weak-event';
 import { Context } from '../context/context.js';
-import { Entity } from './Entity.js';
+import { FileManager } from '../files/FileManager.js';
+import { processValueFiles } from '../files/utils.js';
 import { Disposable } from '../utils/Disposable.js';
+import { Entity } from './Entity.js';
 import { EntityFamilyMetadata } from './EntityMetadata.js';
-import { FileManager } from '../files/FileManager.js';
 import { OperationBatcher } from './OperationBatcher.js';
-import { WeakEvent } from 'weak-event';
-import { processValueFiles } from '../files/utils.js';
 
 enum AbortReason {
 	Reset,
@@ -347,12 +347,11 @@ export class EntityStore extends Disposable {
 			);
 		}
 
-		const operations = this.ctx.patchCreator.createInitialize(processed, oid);
-		if (access) {
-			operations.forEach((op) => {
-				op.authz = access;
-			});
-		}
+		const operations = this.ctx.patchCreator.createInitialize(
+			processed,
+			oid,
+			access,
+		);
 		await this.batcher.commitOperations(operations, {
 			undoable: !!undoable,
 			source: entity,

package/src/entities/OperationBatcher.ts

@@ -11,8 +11,8 @@ import {
 	operationSupersedes,
 } from '@verdant-web/common';
 import { Context } from '../context/context.js';
-import type { EntityStore } from './EntityStore.js';
 import { Entity } from './Entity.js';
+import type { EntityStore } from './EntityStore.js';
 
 const DEFAULT_BATCH_KEY = '@@default';
 
@@ -257,7 +257,11 @@ export class OperationBatcher {
 		return Promise.all(this.batcher.flushAll());
 	};
 
-	private createUndo = async (data: { ops: Operation[]; source?: Entity }) => {
+	private createUndo = async (data: {
+		ops: Operation[];
+		source?: Entity;
+		isRedo?: boolean;
+	}) => {
 		// this can't be done on-demand because we rely on the current
 		// state of the entities to calculate the inverse operations.
 		const inverseOps = await this.getInverseOperations(data);
@@ -268,11 +272,21 @@ export class OperationBatcher {
 			const redo = await this.createUndo({
 				ops: inverseOps,
 				source: data.source,
+				isRedo: true,
 			});
 			// set time to now for all undo operations, they're happening now.
 			for (const op of inverseOps) {
 				op.timestamp = this.ctx.time.now;
 			}
+
+			this.ctx.log(
+				'debug',
+				data.isRedo ? 'Redo' : 'Undo',
+				inverseOps,
+				'\n was \n',
+				data.ops,
+			);
+
 			await this.commitOperations(
 				inverseOps,
 				// undos should not generate their own undo operations