@verdant-web/store 4.3.0 → 4.4.1

package/src/__tests__/entities.test.ts

@@ -2,6 +2,7 @@ import {
 	assert,
 	createRef,
 	EventSubscriber,
+	FixedTimestampProvider,
 	NaiveTimestampProvider,
 	PatchCreator,
 	schema,
@@ -10,6 +11,7 @@ import { describe, expect, it, MockedFunction, vi, vitest } from 'vitest';
 import { WeakEvent } from 'weak-event';
 import { Time } from '../context/Time.js';
 import { EntityFamilyMetadata } from '../entities/EntityMetadata.js';
+import { MARK_FAILED } from '../files/EntityFile.js';
 import { Entity, EntityFile } from '../index.js';
 import { createTestStorage } from './fixtures/testStorage.js';
 
@@ -734,6 +736,7 @@ describe('entities', () => {
 			files: {
 				add: vi.fn(),
 			},
+			weakRef: (v: any) => new WeakRef(v),
 		} as any;
 		const metadataFamily = new EntityFamilyMetadata({
 			ctx: testCtx,
@@ -860,4 +863,312 @@ describe('entities', () => {
 		expect(entity.getSnapshot()).toEqual(entity.getUnprunedSnapshot());
 		expect(entity.deepInvalid).toBe(false);
 	});
+
+	it('should only fire deep change once for a file change even if you get it multiple times', async () => {
+		// addressing a specific bug / lazy thing I did that caused a listener leak
+		const storage = await createTestStorage();
+		const weird = await storage.weirds.put({});
+		const fileList = weird.get('fileList');
+		const file = new window.File(['d(⌐□_□)b'], 'test.txt', {
+			type: 'text/plain',
+		});
+		fileList.push(file);
+		const deepListener = vi.fn();
+		fileList.subscribe('changeDeep', deepListener);
+		fileList.get(0);
+		fileList.get(0);
+		fileList.get(0);
+		// easy way to trigger change
+		fileList.get(0)[MARK_FAILED]();
+		expect(deepListener).toHaveBeenCalledTimes(1);
+	});
+
+	it('should correctly prune deep nested objects', () => {
+		// manually constructing an entity for this test is easiest,
+		// kind of hard to force invalid data otherwise
+		const onPendingOperations = vi.fn();
+		const time = new Time(new FixedTimestampProvider(), 1);
+		// too much junk in here, have to manually pick and choose
+		let subId = 100; // avoid false-flag matching of other ids by choosing an arbitrary start point
+		const testCtx = {
+			globalEvents: new EventSubscriber(),
+			time,
+			log: console.log,
+			patchCreator: new PatchCreator(
+				() => time.now,
+				() => `${subId++}`,
+			),
+			files: {
+				add: vi.fn(),
+			},
+			weakRef: (v: any) => new WeakRef(v),
+		} as any;
+		const metadataFamily = new EntityFamilyMetadata({
+			ctx: testCtx,
+			onPendingOperations,
+			rootOid: 'foos/a',
+		});
+		const entity = new Entity({
+			oid: 'foos/a',
+			schema: {
+				type: 'object',
+				properties: {
+					id: schema.fields.id(),
+					// should prune to an empty list entity
+					listTest: schema.fields.array({
+						items: schema.fields.object({
+							fields: {
+								required: schema.fields.string(),
+							},
+						}),
+					}),
+					objectTest: schema.fields.object({
+						default: () => ({
+							// @ts-ignore - excessive nesting
+							nested: {
+								required: 'foo',
+							},
+						}),
+						fields: {
+							nested: schema.fields.object({
+								fields: {
+									required: schema.fields.string(),
+								},
+							}),
+						},
+					}),
+					// this one will not be present at all.
+					missingTest: schema.fields.object({
+						default: () => ({
+							value: 1,
+						}),
+						fields: {
+							value: schema.fields.number(),
+						},
+					}),
+				},
+			},
+			ctx: testCtx,
+			deleteSelf: vi.fn(),
+			files: {
+				add: vi.fn(),
+			} as any,
+			metadataFamily,
+			storeEvents: {
+				add: new WeakEvent(),
+				replace: new WeakEvent(),
+				resetAll: new WeakEvent(),
+			},
+			readonlyKeys: ['id'],
+		});
+		metadataFamily.addConfirmedData({
+			baselines: [
+				{
+					// an invalid list item
+					oid: 'foos/a:1',
+					snapshot: {},
+					timestamp: time.now,
+				},
+				{
+					oid: 'foos/a:2',
+					snapshot: [createRef('foos/a:1')],
+					timestamp: time.now,
+				},
+				{
+					// an invalid nested object
+					oid: 'foos/a:3',
+					snapshot: {},
+					timestamp: time.now,
+				},
+				{
+					oid: 'foos/a:4',
+					snapshot: { nested: createRef('foos/a:3') },
+					timestamp: time.now,
+				},
+				{
+					oid: 'foos/a',
+					snapshot: {
+						id: 'a',
+						listTest: createRef('foos/a:2'),
+						objectTest: createRef('foos/a:4'),
+					},
+					timestamp: time.now,
+				},
+			],
+		});
+
+		expect(entity.deepInvalid).toBe(true);
+		expect(entity.getSnapshot()).toEqual({
+			id: 'a',
+			listTest: [],
+			// gets the default, pruned
+			objectTest: {
+				nested: {
+					required: 'foo',
+				},
+			},
+			missingTest: {
+				value: 1,
+			},
+		});
+
+		// the kicker...
+		const list = entity.get('listTest');
+		expect(list).toBeInstanceOf(Entity);
+		const obj = entity.get('objectTest');
+		expect(obj).toBeInstanceOf(Entity);
+		const missing = entity.get('missingTest');
+		expect(missing).toBeInstanceOf(Entity);
+
+		// we should be able to make edits to the pruned versions
+		// and they get committed.
+		list.push({ required: 'foo' });
+		expect(entity.getSnapshot()).toEqual({
+			id: 'a',
+			listTest: [{ required: 'foo' }],
+			objectTest: {
+				nested: {
+					required: 'foo',
+				},
+			},
+			missingTest: {
+				value: 1,
+			},
+		});
+		obj.get('nested').set('required', 'bar');
+		expect(entity.getSnapshot()).toEqual({
+			id: 'a',
+			listTest: [{ required: 'foo' }],
+			objectTest: {
+				nested: {
+					required: 'bar',
+				},
+			},
+			missingTest: {
+				value: 1,
+			},
+		});
+		missing.set('value', 2);
+		expect(entity.getSnapshot()).toEqual({
+			id: 'a',
+			listTest: [{ required: 'foo' }],
+			objectTest: {
+				nested: {
+					required: 'bar',
+				},
+				// gets the default, pruned
+			},
+			missingTest: {
+				value: 2,
+			},
+		});
+
+		expect(onPendingOperations).toHaveBeenCalledTimes(5);
+		// when doing those changes above, we see pending operations
+		// added to construct the pruned versions with applied changes
+		function removeExtra(obj: any) {
+			delete obj.timestamp;
+			delete obj.authz;
+			return obj;
+		}
+		// being kind of precious about assertions here since
+		// ordering is not quite as important and some ops create new
+		// oids.
+		const firstCall = onPendingOperations.mock.calls[0][0].map(removeExtra);
+
+		// the required field was set to the default
+		expect(firstCall).toContainEqual({
+			data: {
+				name: 'required',
+				op: 'set',
+				value: 'foo',
+			},
+			oid: 'foos/a:3',
+		});
+
+		// missingTest is created wholesale
+		expect(firstCall).toContainEqual({
+			data: {
+				op: 'set',
+				name: 'missingTest',
+				value: {
+					'@@type': 'ref',
+					id: expect.stringMatching(/^foos\/a:/),
+				},
+			},
+			oid: 'foos/a',
+		});
+		expect(firstCall).toContainEqual({
+			data: {
+				op: 'initialize',
+				value: {
+					value: 1,
+				},
+			},
+			oid: expect.stringMatching(/^foos\/a:/),
+		});
+
+		// we asserted all the contents, now also assert the length
+		expect(firstCall).toHaveLength(3);
+
+		const secondCall = onPendingOperations.mock.calls[1][0].map(removeExtra);
+		// the invalid item was removed from the array
+		expect(secondCall).toContainEqual({
+			data: {
+				op: 'delete',
+			},
+			oid: 'foos/a:1',
+		});
+		expect(secondCall).toContainEqual({
+			data: {
+				op: 'list-remove',
+				only: 'last',
+				value: {
+					'@@type': 'ref',
+					id: 'foos/a:1',
+				},
+			},
+			oid: 'foos/a:2',
+		});
+
+		const thirdCall = onPendingOperations.mock.calls[2][0].map(removeExtra);
+		// we can be more lax here
+		expect(thirdCall).toMatchInlineSnapshot(`
+			[
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": {
+			        "required": "foo",
+			      },
+			    },
+			    "oid": "foos/a:100",
+			  },
+			  {
+			    "data": {
+			      "op": "list-push",
+			      "value": {
+			        "@@type": "ref",
+			        "id": "foos/a:100",
+			      },
+			    },
+			    "oid": "foos/a:2",
+			  },
+			]
+		`);
+
+		const fourthCall = onPendingOperations.mock.calls[3][0].map(removeExtra);
+		expect(fourthCall).toMatchInlineSnapshot(`
+			[
+			  {
+			    "data": {
+			      "name": "required",
+			      "op": "set",
+			      "value": "bar",
+			    },
+			    "oid": "foos/a:3",
+			  },
+			]
+		`);
+	});
 });

package/src/client/Client.ts

@@ -4,6 +4,7 @@ import {
 	EventSubscriber,
 	FileData,
 	Operation,
+	VerdantError,
 } from '@verdant-web/common';
 import { Context } from '../context/context.js';
 import { DocumentManager } from '../entities/DocumentManager.js';
@@ -205,13 +206,20 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 				return await this._entities.addData(data);
 			}
 		} catch (err) {
-			this.context.log('critical', 'Sync failed', err);
+			this.context.log(
+				'critical',
+				'Sync failed. To avoid data corruption, the client will now shut down.',
+				err,
+			);
 			this.emit(
 				'developerError',
-				new Error('Sync failed, see logs or cause', {
-					cause: err,
-				}),
+				new VerdantError(
+					VerdantError.Code.Unexpected,
+					err as any,
+					'Sync failed. To avoid data corruption, the client will now shut down.',
+				),
 			);
+			await this.close();
 			throw err;
 		}
 	};

package/src/entities/Entity.test.ts

@@ -1,16 +1,16 @@
-import { describe, expect, it, vi } from 'vitest';
-import { Entity } from './Entity.js';
-import { EntityFamilyMetadata } from './EntityMetadata.js';
-import { Context } from '../context/context.js';
-import { EntityStoreEvents } from './EntityStore.js';
-import { WeakEvent } from 'weak-event';
-import { FileManager } from '../files/FileManager.js';
 import {
 	NaiveTimestampProvider,
 	PatchCreator,
 	groupPatchesByOid,
 } from '@verdant-web/common';
+import { describe, expect, it, vi } from 'vitest';
+import { WeakEvent } from 'weak-event';
+import { Context } from '../context/context.js';
 import { Time } from '../context/Time.js';
+import { FileManager } from '../files/FileManager.js';
+import { Entity } from './Entity.js';
+import { EntityFamilyMetadata } from './EntityMetadata.js';
+import { EntityStoreEvents } from './EntityStore.js';
 
 describe('Entity', () => {
 	const schema = {
@@ -65,6 +65,7 @@ describe('Entity', () => {
 			log: vi.fn(),
 			time: new Time(time, 1),
 			patchCreator,
+			weakRef: (v: any) => new WeakRef(v),
 		};
 		const entity = new Entity({
 			oid: 'test/1',

package/src/entities/Entity.ts

@@ -10,6 +10,7 @@ import {
 	compareRefs,
 	createFileRef,
 	createRef,
+	createSubOid,
 	getChildFieldSchema,
 	getFieldDefault,
 	hasDefault,
@@ -18,6 +19,7 @@ import {
 	isNullable,
 	isObject,
 	isObjectRef,
+	isPrimitive,
 	isRef,
 	maybeGetOid,
 	memoByKeys,
@@ -25,6 +27,7 @@ import {
 	validateEntityField,
 } from '@verdant-web/common';
 import { Context } from '../context/context.js';
+import { CHILD_FILE_CHANGED } from '../files/EntityFile.js';
 import { FileManager } from '../files/FileManager.js';
 import { processValueFiles } from '../files/utils.js';
 import { ClientWithCollections, EntityFile } from '../index.js';
@@ -112,6 +115,7 @@ export class Entity<
 		files,
 		storeEvents,
 		deleteSelf,
+		fieldPath,
 	}: EntityInit) {
 		super();
 
@@ -122,10 +126,12 @@ export class Entity<
 		this.ctx = ctx;
 		this.files = files;
 		this.schema = schema;
+		this.fieldPath = fieldPath || [];
 		this.entityFamily =
 			childCache ||
 			new EntityCache({
 				initial: [this],
+				ctx,
 			});
 		this.metadataFamily = metadataFamily;
 		this.storeEvents = storeEvents;
@@ -213,8 +219,8 @@ export class Entity<
 			return null as any;
 		}
 
-		this.cachedView = this.isList ? [] : {};
-		assignOid(this.cachedView, this.oid);
+		let newView: any = this.isList ? [] : {};
+		assignOid(newView, this.oid);
 
 		if (Array.isArray(rawView)) {
 			const schema = getChildFieldSchema(this.schema, 0);
@@ -243,7 +249,7 @@ export class Entity<
 
 						// this item will be pruned.
 					} else {
-						this.cachedView.push(child);
+						newView.push(child);
 					}
 				}
 			}
@@ -271,10 +277,10 @@ export class Entity<
 					);
 					if (this.schema.type === 'map') {
 						// it's valid to prune here if it's a map
-						this.cachedView = {};
+						newView = {};
 					} else {
 						// otherwise prune moves upward
-						this.cachedView = null;
+						newView = null;
 					}
 					break;
 				}
@@ -287,28 +293,29 @@ export class Entity<
 						'key:',
 						key,
 					);
-					/**
-					 * PRUNE - this is a prune point. we can't continue
-					 * to render this data. If this is a map, we can ignore
-					 * this value. Otherwise we must prune upward.
-					 * This exits the loop.
-					 */
 					if (this.schema.type !== 'map') {
-						this.cachedView = null;
+						/**
+						 * PRUNE - this is a prune point. we can't continue
+						 * to render this data. If this is a map, we can ignore
+						 * this value. Otherwise we must prune upward.
+						 * This exits the loop.
+						 */
+						newView = null;
 						break;
 					}
 				} else {
 					// special case - rewrite undefined fields to null
 					if (isNullable(schema) && child === undefined) {
-						this.cachedView[key] = null;
+						newView[key] = null;
 					} else {
-						this.cachedView[key] = child;
+						newView[key] = child;
 					}
 				}
 			}
 		}
 
-		return this.cachedView;
+		this.cachedView = newView;
+		return newView;
 	}
 
 	private childIsNull = (child: any) => {
@@ -597,19 +604,16 @@ export class Entity<
 		}
 	};
 
-	private canonizePrunedVersion = () => {
+	private getPruneDiff = () => {
 		const snapshot = this.getSnapshot();
 		const unprunedSnapshot = this.getUnprunedSnapshot();
-		const operations = this.patchCreator.createDiff(
-			unprunedSnapshot,
-			snapshot,
-			{
-				authz: this.access,
-				merge: false,
-			},
-		);
-
-		this.applyPendingOperations(operations);
+		return this.patchCreator.createDiff(unprunedSnapshot, snapshot, {
+			authz: this.access,
+			merge: false,
+		});
+	};
+	private canonizePrunedVersion = () => {
+		this.applyPendingOperations(this.getPruneDiff());
 	};
 
 	private addConfirmedData = (data: EntityStoreEventData) => {
@@ -700,6 +704,10 @@ export class Entity<
 		this.emit('changeDeep', target, ev);
 		this.parent?.deepChange(target, ev);
 	};
+	[CHILD_FILE_CHANGED] = (file: EntityFile) => {
+		// consistent with prior behavior, but kind of arbitrary.
+		this.deepChange(this, { isLocal: false, oid: this.oid });
+	};
 
 	private getChild = (key: any, oid: ObjectIdentifier) => {
 		const schema = getChildFieldSchema(this.schema, key);
@@ -767,16 +775,44 @@ export class Entity<
 				const file = this.files.get(child.id, {
 					downloadRemote: !!fieldSchema.downloadRemote,
 					ctx: this.ctx,
-				});
-
-				// FIXME: this seems bad and inconsistent
-				file.subscribe('change', () => {
-					this.deepChange(this, { isLocal: false, oid: this.oid });
+					parent: this,
 				});
 
 				return file as KeyValue[Key];
 			} else {
-				return this.getChild(key, child.id) as KeyValue[Key];
+				const childEntity = this.getChild(key, child.id);
+				if (childEntity.deepInvalid) {
+					// this child is pruned. materialize a pruned version of
+					// this subtree if possible.
+
+					// special case: lists -- as long as the list itself
+					// is present and valid, it can omit invalid children
+					// selectively rather than fallback to an empty default
+					if (fieldSchema.type === 'array') {
+						return childEntity as KeyValue[Key];
+					}
+					// special case: maps -- similar to lists
+					if (fieldSchema.type === 'map') {
+						return childEntity as KeyValue[Key];
+					}
+					if (isNullable(fieldSchema)) {
+						return null as KeyValue[Key];
+					}
+					if (hasDefault(fieldSchema)) {
+						const unprunedSnapshot = childEntity.getUnprunedSnapshot();
+						const pruneDiff = this.patchCreator.createDiff(
+							unprunedSnapshot,
+							getFieldDefault(fieldSchema),
+							{
+								merge: false,
+								mergeUnknownObjects: true,
+								authz: this.access,
+							},
+						);
+						return this.processPrunedChild(key, childEntity, pruneDiff);
+					}
+				}
+				return childEntity as KeyValue[Key];
 			}
 		} else {
 			// if this is a Map type, a missing child is
@@ -794,16 +830,59 @@ export class Entity<
 					requireDefaults: true,
 				})
 			) {
-				// FIXME: this returns []/{} for arrays and objects, but the contract
-				// of this method should return an Entity for such object fields.
-				// I want to write a test case for this one before attempting to fix
-				// just to be sure the fix works.
-				return getFieldDefault(fieldSchema);
+				if (hasDefault(fieldSchema)) {
+					// primitive fields with defaults are easy.
+					if (isPrimitive(fieldSchema)) {
+						return getFieldDefault(fieldSchema) as KeyValue[Key];
+					}
+
+					// object/list fields are hard.
+					const defaultValue = getFieldDefault(fieldSchema);
+					const prunedFieldOid = createSubOid(this.oid);
+					const pruneDiff = this.patchCreator.createInitialize(
+						defaultValue,
+						prunedFieldOid,
+						this.access,
+					);
+					pruneDiff.push(
+						...this.patchCreator.createSet(
+							this.oid,
+							key,
+							createRef(prunedFieldOid),
+							this.access,
+						),
+					);
+					const childEntity = this.getChild(key, prunedFieldOid);
+					return this.processPrunedChild(key, childEntity, pruneDiff);
+				} else {
+					// failure / hard prune: no way to represent this
+					// data in a valid way exists. the parent entity
+					// is also invalid and this should bubble up.
+					return undefined as KeyValue[Key];
+				}
 			}
 			return child as KeyValue[Key];
 		}
 	};
 
+	private processPrunedChild = (
+		key: any,
+		child: Entity,
+		pruneDiff: Operation[],
+	): any => {
+		this.ctx.log(
+			'warn',
+			'Replacing invalid child object field with ephemeral, valid data',
+			this.oid,
+			key,
+		);
+		const changes = this.metadataFamily.addEphemeralData(pruneDiff);
+		for (const change of changes) {
+			this.change(change);
+		}
+		return child as any;
+	};
+
 	/**
 	 * Gets a value on this entity. If the value is not
 	 * present, it will be set to the provided default
@@ -865,7 +944,7 @@ export class Entity<
 				});
 			}
 		}
-		return processValueFiles(value, this.files.add);
+		return processValueFiles(value, (file) => this.files.add(file, this));
 	};
 
 	private getDeleteMode = (key: any) => {