@verdant-web/store 4.2.0 → 4.4.0

package/src/entities/Entity.ts

@@ -10,6 +10,7 @@ import {
 	compareRefs,
 	createFileRef,
 	createRef,
+	createSubOid,
 	getChildFieldSchema,
 	getFieldDefault,
 	hasDefault,
@@ -17,6 +18,8 @@ import {
 	isFileRef,
 	isNullable,
 	isObject,
+	isObjectRef,
+	isPrimitive,
 	isRef,
 	maybeGetOid,
 	memoByKeys,
@@ -24,6 +27,7 @@ import {
 	validateEntityField,
 } from '@verdant-web/common';
 import { Context } from '../context/context.js';
+import { CHILD_FILE_CHANGED } from '../files/EntityFile.js';
 import { FileManager } from '../files/FileManager.js';
 import { processValueFiles } from '../files/utils.js';
 import { ClientWithCollections, EntityFile } from '../index.js';
@@ -111,6 +115,7 @@ export class Entity<
 		files,
 		storeEvents,
 		deleteSelf,
+		fieldPath,
 	}: EntityInit) {
 		super();
 
@@ -121,10 +126,12 @@ export class Entity<
 		this.ctx = ctx;
 		this.files = files;
 		this.schema = schema;
+		this.fieldPath = fieldPath || [];
 		this.entityFamily =
 			childCache ||
 			new EntityCache({
 				initial: [this],
+				ctx,
 			});
 		this.metadataFamily = metadataFamily;
 		this.storeEvents = storeEvents;
@@ -212,8 +219,8 @@ export class Entity<
 			return null as any;
 		}
 
-		this.cachedView = this.isList ? [] : {};
-		assignOid(this.cachedView, this.oid);
+		let newView: any = this.isList ? [] : {};
+		assignOid(newView, this.oid);
 
 		if (Array.isArray(rawView)) {
 			const schema = getChildFieldSchema(this.schema, 0);
@@ -242,7 +249,7 @@ export class Entity<
 
 						// this item will be pruned.
 					} else {
-						this.cachedView.push(child);
+						newView.push(child);
 					}
 				}
 			}
@@ -270,10 +277,10 @@ export class Entity<
 					);
 					if (this.schema.type === 'map') {
 						// it's valid to prune here if it's a map
-						this.cachedView = {};
+						newView = {};
 					} else {
 						// otherwise prune moves upward
-						this.cachedView = null;
+						newView = null;
 					}
 					break;
 				}
@@ -286,23 +293,29 @@ export class Entity<
 						'key:',
 						key,
 					);
-					/**
-					 * PRUNE - this is a prune point. we can't continue
-					 * to render this data. If this is a map, we can ignore
-					 * this value. Otherwise we must prune upward.
-					 * This exits the loop.
-					 */
 					if (this.schema.type !== 'map') {
-						this.cachedView = null;
+						/**
+						 * PRUNE - this is a prune point. we can't continue
+						 * to render this data. If this is a map, we can ignore
+						 * this value. Otherwise we must prune upward.
+						 * This exits the loop.
+						 */
+						newView = null;
 						break;
 					}
 				} else {
-					this.cachedView[key] = child;
+					// special case - rewrite undefined fields to null
+					if (isNullable(schema) && child === undefined) {
+						newView[key] = null;
+					} else {
+						newView[key] = child;
+					}
 				}
 			}
 		}
 
-		return this.cachedView;
+		this.cachedView = newView;
+		return newView;
 	}
 
 	private childIsNull = (child: any) => {
@@ -325,6 +338,33 @@ export class Entity<
 		return !!this.validate();
 	}
 
+	/**
+	 * Returns true if this or any child is invalid (pruned)
+	 */
+	get deepInvalid(): boolean {
+		if (this.invalid) return true;
+		if (Array.isArray(this.rawView)) {
+			for (let i = 0; i < this.rawView.length; i++) {
+				if (isObjectRef(this.rawView[i])) {
+					const child = this.getChild(i, this.rawView[i].id);
+					if (child.deepInvalid) {
+						return true;
+					}
+				}
+			}
+		} else if (isObject(this.rawView)) {
+			for (const key in this.rawView) {
+				if (isObjectRef(this.rawView[key])) {
+					const child = this.getChild(key, this.rawView[key].id);
+					if (child.deepInvalid) {
+						return true;
+					}
+				}
+			}
+		}
+		return false;
+	}
+
 	get isList() {
 		// have to turn TS off here as our two interfaces both implement
 		// const values for this boolean.
@@ -418,7 +458,7 @@ export class Entity<
 					field: this.schema,
 					value: this.rawView,
 					fieldPath: this.fieldPath,
-					depth: 1,
+					expectRefs: true,
 				}) ?? undefined;
 			return this.validationError;
 		},
@@ -456,6 +496,37 @@ export class Entity<
 		}
 	};
 
+	private rawViewWithMappedChildren = (
+		mapper: (child: Entity | EntityFile) => any,
+	) => {
+		const view = this.rawView;
+		if (!view) {
+			return null;
+		}
+		if (Array.isArray(view)) {
+			const mapped = view.map((value, i) => {
+				if (isRef(value)) {
+					return mapper(this.getChild(i, value.id));
+				} else {
+					return value;
+				}
+			});
+			assignOid(mapped, this.oid);
+			return mapped;
+		} else {
+			const mapped = Object.entries(view).reduce((acc, [key, value]) => {
+				if (isRef(value)) {
+					acc[key as any] = mapper(this.getChild(key, value.id));
+				} else {
+					acc[key as any] = value;
+				}
+				return acc;
+			}, {} as any);
+			assignOid(mapped, this.oid);
+			return mapped;
+		}
+	};
+
 	/**
 	 * A current snapshot of this Entity's data, including nested
 	 * Entities.
@@ -464,10 +535,62 @@ export class Entity<
 		return this.viewWithMappedChildren((child) => child.getSnapshot());
 	};
 
+	/**
+	 * A snapshot of this Entity with unpruned (invalid) data. This will
+	 * not conform to the entity schema and should be used carefully.
+	 *
+	 * Can be used to inspect or recover invalid, pruned data not
+	 * otherwise accessible.
+	 */
+	getUnprunedSnapshot = (): any => {
+		return this.rawViewWithMappedChildren((child) => {
+			if (child instanceof EntityFile) return child.getSnapshot();
+			return child.getUnprunedSnapshot();
+		});
+	};
+
 	// change management methods (internal use only)
 	private addPendingOperations = (operations: Operation[]) => {
-		this.ctx.log('debug', 'Entity: adding pending operations', this.oid);
+		this.ctx.log(
+			'debug',
+			'Entity: adding pending operations',
+			this.oid,
+			operations,
+		);
 
+		// special case -- if this entity is pruned, any changes we apply to it
+		// will be in relation to 'exposed' pruned data, not the 'real world'
+		// data that's backing it. That means those changes will produce unexpected
+		// or further invalid results. To avoid this, we basically stamp in the
+		// pruned version of this entity before proceeding.
+		//
+		// as an example of a failure mode without this check, consider a list:
+		// [1, 2, <pruned>, 4, 5]
+		// the user sees: [1, 2, 4, 5]
+		// when they try to replace the item at index 2 with "0" (they see "4"), they
+		// actually replace the invisible pruned item, resulting in [1, 2, 0, 4, 5]
+		// being the result when they expected [1, 2, 0, 5].
+		//
+		// To "stamp" the data before applying user changes, we diff the snapshot
+		// (which is the pruned version) with the current state of the entity.
+		if (this.deepInvalid) {
+			this.ctx.log(
+				'warn',
+				'Changes are being applied to a pruned entity. This means that the pruned version is being treated as the new baseline and any pruned invalid data is lost.',
+				this.oid,
+			);
+			this.canonizePrunedVersion();
+		}
+
+		this.applyPendingOperations(operations);
+	};
+
+	// naming is fuzzy here, but this method was split out from
+	// addPendingOperations since that method also conditionally canonizes
+	// the pruned snapshot, and I wanted to keep the actual insertion of
+	// the ops in one place, so leaving it as part of addPendingOperations
+	// would introduce infinite recursion when canonizing.
+	private applyPendingOperations = (operations: Operation[]) => {
 		// apply authz to all operations
 		if (this.access) {
 			for (const op of operations) {
@@ -481,6 +604,18 @@ export class Entity<
 		}
 	};
 
+	private getPruneDiff = () => {
+		const snapshot = this.getSnapshot();
+		const unprunedSnapshot = this.getUnprunedSnapshot();
+		return this.patchCreator.createDiff(unprunedSnapshot, snapshot, {
+			authz: this.access,
+			merge: false,
+		});
+	};
+	private canonizePrunedVersion = () => {
+		this.applyPendingOperations(this.getPruneDiff());
+	};
+
 	private addConfirmedData = (data: EntityStoreEventData) => {
 		this.ctx.log('debug', 'Entity: adding confirmed data', this.oid);
 		const changes = this.metadataFamily.addConfirmedData(data);
@@ -569,6 +704,10 @@ export class Entity<
 		this.emit('changeDeep', target, ev);
 		this.parent?.deepChange(target, ev);
 	};
+	[CHILD_FILE_CHANGED] = (file: EntityFile) => {
+		// consistent with prior behavior, but kind of arbitrary.
+		this.deepChange(this, { isLocal: false, oid: this.oid });
+	};
 
 	private getChild = (key: any, oid: ObjectIdentifier) => {
 		const schema = getChildFieldSchema(this.schema, key);
@@ -636,16 +775,44 @@ export class Entity<
 				const file = this.files.get(child.id, {
 					downloadRemote: !!fieldSchema.downloadRemote,
 					ctx: this.ctx,
-				});
-
-				// FIXME: this seems bad and inconsistent
-				file.subscribe('change', () => {
-					this.deepChange(this, { isLocal: false, oid: this.oid });
+					parent: this,
 				});
 
 				return file as KeyValue[Key];
 			} else {
-				return this.getChild(key, child.id) as KeyValue[Key];
+				const childEntity = this.getChild(key, child.id);
+				if (childEntity.deepInvalid) {
+					// this child is pruned. materialize a pruned version of
+					// this subtree if possible.
+
+					// special case: lists -- as long as the list itself
+					// is present and valid, it can omit invalid children
+					// selectively rather than fallback to an empty default
+					if (fieldSchema.type === 'array') {
+						return childEntity as KeyValue[Key];
+					}
+					// special case: maps -- similar to lists
+					if (fieldSchema.type === 'map') {
+						return childEntity as KeyValue[Key];
+					}
+					if (isNullable(fieldSchema)) {
+						return null as KeyValue[Key];
+					}
+					if (hasDefault(fieldSchema)) {
+						const unprunedSnapshot = childEntity.getUnprunedSnapshot();
+						const pruneDiff = this.patchCreator.createDiff(
+							unprunedSnapshot,
+							getFieldDefault(fieldSchema),
+							{
+								merge: false,
+								mergeUnknownObjects: true,
+								authz: this.access,
+							},
+						);
+						return this.processPrunedChild(key, childEntity, pruneDiff);
+					}
+				}
+				return childEntity as KeyValue[Key];
 			}
 		} else {
 			// if this is a Map type, a missing child is
@@ -663,16 +830,59 @@ export class Entity<
 					requireDefaults: true,
 				})
 			) {
-				// FIXME: this returns []/{} for arrays and objects, but the contract
-				// of this method should return an Entity for such object fields.
-				// I want to write a test case for this one before attempting to fix
-				// just to be sure the fix works.
-				return getFieldDefault(fieldSchema);
+				if (hasDefault(fieldSchema)) {
+					// primitive fields with defaults are easy.
+					if (isPrimitive(fieldSchema)) {
+						return getFieldDefault(fieldSchema) as KeyValue[Key];
+					}
+
+					// object/list fields are hard.
+					const defaultValue = getFieldDefault(fieldSchema);
+					const prunedFieldOid = createSubOid(this.oid);
+					const pruneDiff = this.patchCreator.createInitialize(
+						defaultValue,
+						prunedFieldOid,
+						this.access,
+					);
+					pruneDiff.push(
+						...this.patchCreator.createSet(
+							this.oid,
+							key,
+							createRef(prunedFieldOid),
+							this.access,
+						),
+					);
+					const childEntity = this.getChild(key, prunedFieldOid);
+					return this.processPrunedChild(key, childEntity, pruneDiff);
+				} else {
+					// failure / hard prune: no way to represent this
+					// data in a valid way exists. the parent entity
+					// is also invalid and this should bubble up.
+					return undefined as KeyValue[Key];
+				}
 			}
 			return child as KeyValue[Key];
 		}
 	};
 
+	private processPrunedChild = (
+		key: any,
+		child: Entity,
+		pruneDiff: Operation[],
+	): any => {
+		this.ctx.log(
+			'warn',
+			'Replacing invalid child object field with ephemeral, valid data',
+			this.oid,
+			key,
+		);
+		const changes = this.metadataFamily.addEphemeralData(pruneDiff);
+		for (const change of changes) {
+			this.change(change);
+		}
+		return child as any;
+	};
+
 	/**
 	 * Gets a value on this entity. If the value is not
 	 * present, it will be set to the provided default
@@ -734,7 +944,7 @@ export class Entity<
 				});
 			}
 		}
-		return processValueFiles(value, this.files.add);
+		return processValueFiles(value, (file) => this.files.add(file, this));
 	};
 
 	private getDeleteMode = (key: any) => {
@@ -927,7 +1137,7 @@ export class Entity<
 		this.addPendingOperations(
 			this.patchCreator.createDiff(this.getSnapshot(), changes, {
 				mergeUnknownObjects: !replaceSubObjects,
-				defaultUndefined: merge,
+				merge,
 			}),
 		);
 	};

package/src/entities/EntityCache.ts

@@ -1,24 +1,25 @@
-import { Entity, EntityInit } from './Entity.js';
-import { EntityFile } from '../files/EntityFile.js';
 import { ObjectIdentifier } from '@verdant-web/common';
+import { Context } from '../internal.js';
+import { Entity, EntityInit } from './Entity.js';
 
 export class EntityCache {
-	private cache = new Map<string, Entity | EntityFile>();
+	private ctx: Context;
+	private cache = new Map<string, WeakRef<Entity>>();
 
-	constructor({ initial }: { initial?: Entity[] } = {}) {
+	constructor({ initial, ctx }: { initial?: Entity[]; ctx: Context }) {
+		this.ctx = ctx;
 		if (initial) {
 			for (const entity of initial) {
-				this.cache.set(entity.oid, entity);
+				this.cache.set(entity.oid, ctx.weakRef(entity));
 			}
 		}
 	}
 
 	get = (init: EntityInit): Entity => {
-		if (this.cache.has(init.oid)) {
-			return this.cache.get(init.oid)! as Entity;
-		}
+		const cached = this.getCached(init.oid);
+		if (cached) return cached;
 		const entity = new Entity(init);
-		this.cache.set(init.oid, entity);
+		this.cache.set(init.oid, this.ctx.weakRef(entity));
 		return entity;
 	};
 
@@ -27,6 +28,15 @@ export class EntityCache {
 	};
 
 	getCached = (oid: string) => {
-		return this.cache.get(oid);
+		if (this.cache.has(oid)) {
+			const ref = this.cache.get(oid);
+			const derefed = ref?.deref();
+			if (derefed) {
+				return derefed as Entity;
+			} else {
+				this.cache.delete(oid);
+			}
+		}
+		return null;
 	};
 }

package/src/entities/EntityMetadata.ts

@@ -1,4 +1,5 @@
 import {
+	AuthorizationKey,
 	DocumentBaseline,
 	ObjectIdentifier,
 	Operation,
@@ -20,7 +21,7 @@ export type EntityMetadataView = {
 	empty: boolean;
 	updatedAt: number;
 	latestTimestamp: string | null;
-	authz?: string;
+	authz?: AuthorizationKey;
 };
 
 export class EntityMetadata {
@@ -28,6 +29,14 @@ export class EntityMetadata {
 	private baseline: DocumentBaseline | null = null;
 	// these must be kept in timestamp order.
 	private confirmedOperations: Operation[] = [];
+	// operations applied locally, but not sent to persistence
+	// until 'committed' by pending operations. this powers the
+	// self-healing pruning system, which injects these ephemeral
+	// operations to materialize pruned 'fixed' data in place of
+	// 'real' invalid data so the user can keep using the app. as
+	// soon as the user makes modifications to the entity, this
+	// ephemeral pruned data is applied underneath it.
+	private ephemeralOperations?: Operation[] = [];
 	private pendingOperations: Operation[] = [];
 	readonly oid;
 
@@ -89,19 +98,30 @@ export class EntityMetadata {
 			authz = confirmedResult.authz;
 		}
 
+		const ephemeralResult =
+			!this.ephemeralOperations || omitPending
+				? confirmedResult
+				: this.applyOperations(
+						confirmedResult.view,
+						confirmedResult.deleted,
+						this.ephemeralOperations,
+						confirmedResult.latestTimestamp,
+						null,
+					);
+
 		const pendingResult = omitPending
 			? confirmedResult
 			: this.applyOperations(
-					confirmedResult.view,
-					confirmedResult.deleted,
+					ephemeralResult.view,
+					ephemeralResult.deleted,
 					// now we're applying pending operations
 					this.pendingOperations,
 					// keep our latest timestamp up to date
-					confirmedResult.latestTimestamp,
+					ephemeralResult.latestTimestamp,
 					// we don't use after for pending ops, they're all
 					// logically in the future
 					null,
-			  );
+				);
 		if (pendingResult.authz) {
 			authz = pendingResult.authz;
 		}
@@ -199,11 +219,22 @@ export class EntityMetadata {
 	};
 
 	addPendingOperation = (operation: Operation) => {
-		// check to see if new operation supersedes the previous one
-		// we can assume pending ops are always newer
 		this.pendingOperations.push(operation);
 	};
 
+	addEphemeralOperation = (operation: Operation) => {
+		if (!this.ephemeralOperations) {
+			this.ephemeralOperations = [];
+		}
+		this.ephemeralOperations.push(operation);
+	};
+
+	clearEphemeralOperations = () => {
+		const old = this.ephemeralOperations;
+		this.ephemeralOperations = [];
+		return old;
+	};
+
 	discardPendingOperation = (operation: Operation) => {
 		this.pendingOperations = this.pendingOperations.filter(
 			(op) => op.timestamp !== operation.timestamp,
@@ -221,10 +252,10 @@ export class EntityMetadata {
 		latestTimestamp: string | null;
 		deleted: boolean;
 		futureSeen: string | undefined;
-		authz?: string;
+		authz?: AuthorizationKey;
 	} => {
 		let futureSeen: string | undefined = undefined;
-		let authz: string | undefined = undefined;
+		let authz: AuthorizationKey | undefined = undefined;
 
 		const now = this.ctx.time.now;
 		for (const op of operations) {
@@ -345,6 +376,10 @@ export class EntityFamilyMetadata {
 	 * local changes are usually handled, as a list.
 	 */
 	addPendingData = (operations: Operation[]) => {
+		// when pending data is applied, we go ahead and
+		// write all ephemeral changes first.
+		this.flushAllEphemeral();
+
 		const changes: Record<ObjectIdentifier, EntityChange> = {};
 		for (const op of operations) {
 			this.get(op.oid).addPendingOperation(op);
@@ -354,6 +389,31 @@ export class EntityFamilyMetadata {
 		return Object.values(changes);
 	};
 
+	private ephemeralMemo = new Array<Operation>();
+	private flushAllEphemeral = () => {
+		for (const ent of this.entities.values()) {
+			const ops = ent.clearEphemeralOperations();
+			if (ops) {
+				this.ephemeralMemo.push(...ops);
+			}
+		}
+		if (this.ephemeralMemo.length) {
+			const ephemeralCopy = this.ephemeralMemo.slice();
+			// must clear this first to avoid infinite recursion
+			this.ephemeralMemo.length = 0;
+			this.addPendingData(ephemeralCopy);
+		}
+	};
+
+	addEphemeralData = (operations: Operation[]) => {
+		const changes: Record<ObjectIdentifier, EntityChange> = {};
+		for (const op of operations) {
+			this.get(op.oid).addEphemeralOperation(op);
+			changes[op.oid] ??= { oid: op.oid, isLocal: true };
+		}
+		return Object.values(changes);
+	};
+
 	replaceAllData = ({
 		operations = {},
 		baselines = [],

package/src/entities/EntityStore.ts

@@ -1,5 +1,7 @@
 import {
+	AuthorizationKey,
 	DocumentBaseline,
+	FileData,
 	ObjectIdentifier,
 	Operation,
 	StorageObjectFieldSchema,
@@ -12,16 +14,15 @@ import {
 	groupPatchesByRootOid,
 	isRootOid,
 	removeOidsFromAllSubObjects,
-	AuthorizationKey,
 } from '@verdant-web/common';
+import { WeakEvent } from 'weak-event';
 import { Context } from '../context/context.js';
-import { Entity } from './Entity.js';
+import { FileManager } from '../files/FileManager.js';
+import { processValueFiles } from '../files/utils.js';
 import { Disposable } from '../utils/Disposable.js';
+import { Entity } from './Entity.js';
 import { EntityFamilyMetadata } from './EntityMetadata.js';
-import { FileManager } from '../files/FileManager.js';
 import { OperationBatcher } from './OperationBatcher.js';
-import { WeakEvent } from 'weak-event';
-import { processValueFiles } from '../files/utils.js';
 
 enum AbortReason {
 	Reset,
@@ -335,7 +336,8 @@ export class EntityStore extends Disposable {
 		// remove any OID associations from the initial data
 		removeOidsFromAllSubObjects(initial);
 		// grab files and replace them with refs
-		const processed = processValueFiles(initial, this.files.add);
+		const fileRefs: FileData[] = [];
+		const processed = processValueFiles(initial, fileRefs.push.bind(fileRefs));
 
 		assignOid(processed, oid);
 
@@ -346,13 +348,14 @@ export class EntityStore extends Disposable {
 				`Could not put new document: no schema exists for collection ${collection}`,
 			);
 		}
+		// add files with entity as parent
+		fileRefs.forEach((file) => this.files.add(file, entity));
 
-		const operations = this.ctx.patchCreator.createInitialize(processed, oid);
-		if (access) {
-			operations.forEach((op) => {
-				op.authz = access;
-			});
-		}
+		const operations = this.ctx.patchCreator.createInitialize(
+			processed,
+			oid,
+			access,
+		);
 		await this.batcher.commitOperations(operations, {
 			undoable: !!undoable,
 			source: entity,