@verdant-web/common 1.11.1 → 1.13.0

package/src/operation.test.ts

@@ -18,7 +18,13 @@ import {
 
 function createClock(init = 0) {
 	let i = init;
-	return () => (i++).toString();
+	const clock = () => (i++).toString();
+	Object.defineProperty(clock, 'current', {
+		get() {
+			return i;
+		},
+	});
+	return clock;
 }
 
 describe('creating diff patch operations', () => {
@@ -28,7 +34,7 @@ describe('creating diff patch operations', () => {
 			assignOid(from, 'test/a');
 			const to = { foo: 'bar', baz: 'pop' };
 			assignOid(to, 'test/a');
-			const ops = diffToPatches(from, to, createClock(), []);
+			const ops = diffToPatches(from, to, createClock());
 			expect(ops).toMatchInlineSnapshot(`
 				[
 				  {
@@ -70,7 +76,7 @@ describe('creating diff patch operations', () => {
 				},
 				'test/a',
 			);
-			const patches = diffToPatches(from, to, createClock(), []);
+			const patches = diffToPatches(from, to, createClock());
 			expect(patches).toMatchInlineSnapshot(`
 				[
 				  {
@@ -118,7 +124,7 @@ describe('creating diff patch operations', () => {
 			};
 			assignOid(to, 'test/a');
 			assignOid(to.baz, 'test/a:1');
-			const patches = diffToPatches(from, to, createClock(), []);
+			const patches = diffToPatches(from, to, createClock());
 			expect(patches).toMatchInlineSnapshot(`
 				[
 				  {
@@ -166,7 +172,7 @@ describe('creating diff patch operations', () => {
 			};
 			assignOid(to, 'test/a');
 			assignOid(to.baz, 'test/a:0');
-			const patches = diffToPatches(from, to, createClock(), []);
+			const patches = diffToPatches(from, to, createClock());
 			expect(patches).toEqual([]);
 		});
 		it('patches sub-objects with same identity', () => {
@@ -182,7 +188,7 @@ describe('creating diff patch operations', () => {
 			};
 			assignOid(to, 'test/a');
 			assignOid(to.baz, 'test/a:0');
-			const patches = diffToPatches(from, to, createClock(), []);
+			const patches = diffToPatches(from, to, createClock());
 			expect(patches).toMatchInlineSnapshot(`
 				[
 				  {
@@ -208,18 +214,10 @@ describe('creating diff patch operations', () => {
 				foo: 'bip',
 			};
 			assignOid(to, 'test/a');
-			const patches = diffToPatches(
-				from,
-				to,
-				createClock(),
-				[],
-				undefined,
-				[],
-				{
-					defaultUndefined: true,
-					mergeUnknownObjects: true,
-				},
-			);
+			const patches = diffToPatches(from, to, createClock(), undefined, [], {
+				defaultUndefined: true,
+				mergeUnknownObjects: true,
+			});
 			expect(patches).toMatchInlineSnapshot(`
 				[
 				  {
@@ -303,7 +301,7 @@ describe('creating diff patch operations', () => {
 		assignOid(to.baz, 'test/a:3');
 		assignOid(to.baz[0], 'test/a:4');
 		assignOid(to.baz[1], 'test/a:5');
-		expect(diffToPatches(from, to, createClock(), [])).toMatchInlineSnapshot(`
+		expect(diffToPatches(from, to, createClock())).toMatchInlineSnapshot(`
 			[
 			  {
 			    "data": {
@@ -400,7 +398,7 @@ describe('creating diff patch operations', () => {
 			],
 		};
 		expect(
-			diffToPatches(from, to, createClock(), [], createClock(5), [], {
+			diffToPatches(from, to, createClock(), createClock(5), [], {
 				mergeUnknownObjects: true,
 			}),
 		).toMatchInlineSnapshot(`
@@ -411,7 +409,7 @@ describe('creating diff patch operations', () => {
 			      "index": 2,
 			      "op": "list-delete",
 			    },
-			    "oid": "test/a.foo.bar:1",
+			    "oid": "test/a:1",
 			    "timestamp": "0",
 			  },
 			  {
@@ -421,7 +419,7 @@ describe('creating diff patch operations', () => {
 			        0,
 			      ],
 			    },
-			    "oid": "test/a.foo.bop:5",
+			    "oid": "test/a:5",
 			    "timestamp": "1",
 			  },
 			  {
@@ -430,10 +428,10 @@ describe('creating diff patch operations', () => {
 			      "op": "set",
 			      "value": {
 			        "@@type": "ref",
-			        "id": "test/a.foo.bop:5",
+			        "id": "test/a:5",
 			      },
 			    },
-			    "oid": "test/a.foo:0",
+			    "oid": "test/a:0",
 			    "timestamp": "2",
 			  },
 			  {
@@ -442,7 +440,7 @@ describe('creating diff patch operations', () => {
 			      "op": "set",
 			      "value": false,
 			    },
-			    "oid": "test/a.baz.#:3",
+			    "oid": "test/a:3",
 			    "timestamp": "3",
 			  },
 			  {
@@ -452,7 +450,7 @@ describe('creating diff patch operations', () => {
 			        "corge": false,
 			      },
 			    },
-			    "oid": "test/a.baz.#:6",
+			    "oid": "test/a:6",
 			    "timestamp": "4",
 			  },
 			  {
@@ -461,10 +459,10 @@ describe('creating diff patch operations', () => {
 			      "op": "set",
 			      "value": {
 			        "@@type": "ref",
-			        "id": "test/a.baz.#:6",
+			        "id": "test/a:6",
 			      },
 			    },
-			    "oid": "test/a.baz:2",
+			    "oid": "test/a:2",
 			    "timestamp": "5",
 			  },
 			]
@@ -488,7 +486,7 @@ describe('creating diff patch operations', () => {
 			bar: [createFileRef('def456')],
 		};
 		expect(
-			diffToPatches(from, to, createClock(), [], createClock(5), [], {
+			diffToPatches(from, to, createClock(), createClock(5), [], {
 				mergeUnknownObjects: true,
 			}),
 		).toMatchInlineSnapshot(`
@@ -502,7 +500,7 @@ describe('creating diff patch operations', () => {
 			        "id": "abc456",
 			      },
 			    },
-			    "oid": "test/a.foo:0",
+			    "oid": "test/a:0",
 			    "timestamp": "0",
 			  },
 			  {
@@ -511,12 +509,162 @@ describe('creating diff patch operations', () => {
 			      "index": 1,
 			      "op": "list-delete",
 			    },
-			    "oid": "test/a.bar:1",
+			    "oid": "test/a:1",
 			    "timestamp": "1",
 			  },
 			]
 		`);
 	});
+
+	it('should assign a new OID to objects which had an incompatible existing OID', () => {
+		const from = {
+			foo: {
+				bar: [1, 2, 3],
+			},
+		};
+		assignOid(from, 'test/a');
+		assignOidsToAllSubObjects(from, createClock());
+		const to = {
+			foo: {
+				bar: [1, 2],
+			},
+		};
+		assignOid(to, 'test/a');
+		assignOid(to.foo, 'uff/b');
+		assignOidsToAllSubObjects(to.foo, createClock());
+		expect(
+			diffToPatches(from, to, createClock(), createClock(5), [], {
+				mergeUnknownObjects: true,
+			}),
+		).toMatchInlineSnapshot(`
+			[
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": [
+			        1,
+			        2,
+			      ],
+			    },
+			    "oid": "test/a:6",
+			    "timestamp": "0",
+			  },
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": {
+			        "bar": {
+			          "@@type": "ref",
+			          "id": "test/a:6",
+			        },
+			      },
+			    },
+			    "oid": "test/a:5",
+			    "timestamp": "1",
+			  },
+			  {
+			    "data": {
+			      "name": "foo",
+			      "op": "set",
+			      "value": {
+			        "@@type": "ref",
+			        "id": "test/a:5",
+			      },
+			    },
+			    "oid": "test/a",
+			    "timestamp": "2",
+			  },
+			  {
+			    "data": {
+			      "op": "delete",
+			    },
+			    "oid": "test/a:0",
+			    "timestamp": "3",
+			  },
+			]
+		`);
+	});
+
+	// an edge case - if a subobject which contains nested objects changes identity,
+	// this gets written as a new init patch tree - we want to ensure all init subobjects
+	// in that tree have compatible OIDs.
+	it('should assign a new OID to objects which have an incompatible OID when parent identity changes', () => {
+		const from = {
+			bar: [],
+		};
+		assignOid(from, 'test/a');
+		assignOid(from.bar, 'test/random');
+		const to = {
+			bar: [{ baz: 1 }, { baz: 2 }],
+		};
+		assignOid(to, 'test/a');
+		assignOidsToAllSubObjects(to, createClock());
+		assignOid(to.bar[0], 'uff/b');
+		expect(
+			diffToPatches(from, to, createClock(), createClock(5), [], {
+				mergeUnknownObjects: true,
+			}),
+		).toMatchInlineSnapshot(`
+			[
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": {
+			        "baz": 1,
+			      },
+			    },
+			    "oid": "test/a:5",
+			    "timestamp": "0",
+			  },
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": {
+			        "baz": 2,
+			      },
+			    },
+			    "oid": "test/a:2",
+			    "timestamp": "1",
+			  },
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": [
+			        {
+			          "@@type": "ref",
+			          "id": "test/a:5",
+			        },
+			        {
+			          "@@type": "ref",
+			          "id": "test/a:2",
+			        },
+			      ],
+			    },
+			    "oid": "test/a:0",
+			    "timestamp": "2",
+			  },
+			  {
+			    "data": {
+			      "name": "bar",
+			      "op": "set",
+			      "value": {
+			        "@@type": "ref",
+			        "id": "test/a:0",
+			      },
+			    },
+			    "oid": "test/a",
+			    "timestamp": "3",
+			  },
+			  {
+			    "data": {
+			      "op": "delete",
+			    },
+			    "oid": "test/random",
+			    "timestamp": "4",
+			  },
+			]
+		`);
+	});
 });
 
 describe('substituting refs with objects', () => {
@@ -730,7 +878,7 @@ describe('creating patches from initial state', () => {
 			        "bar": "baz",
 			      },
 			    },
-			    "oid": "test/a.foo:0",
+			    "oid": "test/a:0",
 			    "timestamp": "0",
 			  },
 			  {
@@ -740,7 +888,7 @@ describe('creating patches from initial state', () => {
 			        "corge": "grault",
 			      },
 			    },
-			    "oid": "test/a.qux.#:2",
+			    "oid": "test/a:2",
 			    "timestamp": "1",
 			  },
 			  {
@@ -750,7 +898,7 @@ describe('creating patches from initial state', () => {
 			        "oof": 1,
 			      },
 			    },
-			    "oid": "test/a.qux.#.bin:4",
+			    "oid": "test/a:4",
 			    "timestamp": "2",
 			  },
 			  {
@@ -759,11 +907,11 @@ describe('creating patches from initial state', () => {
 			      "value": {
 			        "bin": {
 			          "@@type": "ref",
-			          "id": "test/a.qux.#.bin:4",
+			          "id": "test/a:4",
 			        },
 			      },
 			    },
-			    "oid": "test/a.qux.#:3",
+			    "oid": "test/a:3",
 			    "timestamp": "3",
 			  },
 			  {
@@ -772,15 +920,15 @@ describe('creating patches from initial state', () => {
 			      "value": [
 			        {
 			          "@@type": "ref",
-			          "id": "test/a.qux.#:2",
+			          "id": "test/a:2",
 			        },
 			        {
 			          "@@type": "ref",
-			          "id": "test/a.qux.#:3",
+			          "id": "test/a:3",
 			        },
 			      ],
 			    },
-			    "oid": "test/a.qux:1",
+			    "oid": "test/a:1",
 			    "timestamp": "4",
 			  },
 			  {
@@ -789,11 +937,11 @@ describe('creating patches from initial state', () => {
 			      "value": {
 			        "foo": {
 			          "@@type": "ref",
-			          "id": "test/a.foo:0",
+			          "id": "test/a:0",
 			        },
 			        "qux": {
 			          "@@type": "ref",
-			          "id": "test/a.qux:1",
+			          "id": "test/a:1",
 			        },
 			      },
 			    },

package/src/operation.ts

@@ -1,23 +1,22 @@
 import { DocumentBaseline } from './baseline.js';
 import { FileRef } from './files.js';
 import {
+	areOidsRelated,
 	assignOid,
 	assignOidsToAllSubObjects,
 	createRef,
-	decomposeOid,
+	createSubOid,
 	ensureOid,
 	getOid,
 	getOidRoot,
 	isOidKey,
-	KeyPath,
 	maybeGetOid,
 	normalize,
 	ObjectIdentifier,
-	OID_KEY,
 	removeOid,
 } from './oids.js';
 import { compareRefs, isRef } from './refs.js';
-import { isObject, assert, cloneDeep, findLastIndex } from './utils.js';
+import { assert, cloneDeep, findLastIndex, isObject } from './utils.js';
 
 // export type ObjectIdentifier<
 // 	CollectionName extends string = string,
@@ -161,7 +160,6 @@ export function diffToPatches<T extends { [key: string]: any } | any[]>(
 	from: T,
 	to: T,
 	getNow: () => string,
-	keyPath: KeyPath,
 	createSubId?: () => string,
 	patches: Operation[] = [],
 	options: {
@@ -196,20 +194,34 @@ export function diffToPatches<T extends { [key: string]: any } | any[]>(
 			}
 		} else {
 			const oldValueOid = maybeGetOid(oldValue);
-			let valueOid;
-			if (!options.mergeUnknownObjects) {
-				valueOid = ensureOid(value, oid, key, createSubId);
+			let valueOid = maybeGetOid(value);
+			// the user supplied an object which was already assigned an OID,
+			// but that OID is from a separate entity: this object must
+			// be cloned to a new identity.
+			if (valueOid && !areOidsRelated(oid, valueOid)) {
+				value = cloneDeep(value, false);
+				valueOid = createSubOid(oid, createSubId);
+				assignOid(value, valueOid);
+			} else if (!options.mergeUnknownObjects) {
+				valueOid = ensureOid(value, oid, createSubId);
 			} else {
 				// if merge unknown objects is requested, we copy the previous value's oid
 				// to any mirrored new value if it doesn't have one assigned already.
-				valueOid = maybeGetOid(value);
 				if (!valueOid && oldValueOid) {
 					assignOid(value, oldValueOid);
 					valueOid = oldValueOid;
 				} else {
-					valueOid = ensureOid(value, oid, key, createSubId);
+					valueOid = ensureOid(value, oid, createSubId);
 				}
 			}
+			assert(
+				!!valueOid,
+				'Error: no value OID was resolved during diff. This is a bug in Verdant.',
+			);
+			assert(
+				areOidsRelated(oid, valueOid),
+				`Error: value OID ${valueOid} is not related to parent OID ${oid}. This is a bug in Verdant.`,
+			);
 
 			if (oldValue === undefined || valueOid !== oldValueOid) {
 				// first case: previous value exists but the OIDs are different,
@@ -239,15 +251,7 @@ export function diffToPatches<T extends { [key: string]: any } | any[]>(
 			} else {
 				// third case: OIDs are the same, meaning the identity is the same,
 				// and we must diff the objects
-				diffToPatches(
-					oldValue,
-					value,
-					getNow,
-					[...keyPath, key],
-					createSubId,
-					patches,
-					options,
-				);
+				diffToPatches(oldValue, value, getNow, createSubId, patches, options);
 			}
 		}
 	}

package/src/patch.ts

@@ -2,7 +2,7 @@
  * High-level patch creation for use with complex nested objects.
  */
 
-import { createRef, createSubOid, KeyPath, ObjectIdentifier } from './oids.js';
+import { createRef, createSubOid, ObjectIdentifier } from './oids.js';
 import {
 	diffToPatches,
 	initialToPatches,
@@ -22,18 +22,9 @@ export class PatchCreator {
 	createDiff = (
 		from: any,
 		to: any,
-		keyPath: KeyPath,
 		options: { mergeUnknownObjects?: boolean; defaultUndefined?: boolean } = {},
 	) => {
-		return diffToPatches(
-			from,
-			to,
-			this.getNow,
-			keyPath,
-			this.createSubId,
-			[],
-			options,
-		);
+		return diffToPatches(from, to, this.getNow, this.createSubId, [], options);
 	};
 
 	createInitialize = (obj: any, oid: ObjectIdentifier) => {
@@ -44,7 +35,6 @@ export class PatchCreator {
 		oid: ObjectIdentifier,
 		key: PropertyName,
 		value: any,
-		keyPath: KeyPath,
 	): Operation[] => {
 		// incoming value must be normalized. if it's not a primitive, it and all sub-objects
 		// must be created
@@ -61,7 +51,7 @@ export class PatchCreator {
 				},
 			];
 		} else {
-			const itemOid = createSubOid(oid, key, this.createSubId);
+			const itemOid = createSubOid(oid, this.createSubId);
 			return [
 				// since we're setting a complex nested object, we can initialize it wholesale.
 				// no diffing to do.
@@ -106,7 +96,7 @@ export class PatchCreator {
 				},
 			];
 		} else {
-			const itemOid = createSubOid(oid, 0, this.createSubId);
+			const itemOid = createSubOid(oid, this.createSubId);
 			return [
 				...initialToPatches(value, itemOid, this.getNow),
 				{
@@ -165,7 +155,7 @@ export class PatchCreator {
 				},
 			];
 		} else {
-			const itemOid = createSubOid(oid, index, this.createSubId);
+			const itemOid = createSubOid(oid, this.createSubId);
 			return [
 				...initialToPatches(value, itemOid, this.getNow),
 				{

package/src/schema/defaults.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { getFieldDefault } from './defaults.js';
+import { getFieldDefault } from './fields.js';
 import { assignOid, createOid, getOid, maybeGetOid } from '../oids.js';
 
 describe('getting field defaults', () => {
@@ -12,7 +12,7 @@ describe('getting field defaults', () => {
 			},
 		} as const;
 		const first = getFieldDefault(field);
-		assignOid(first, createOid('test', '1', []));
+		assignOid(first, createOid('test', '1'));
 		const second = getFieldDefault(field);
 		expect(maybeGetOid(second)).toBe(undefined);
 	});

package/src/schema/{defaults.ts → fields.ts}

@@ -1,13 +1,25 @@
 import { LEGACY_OID_KEY, OID_KEY } from '../oids.js';
+import { isObject } from '../utils.js';
 import type { StorageFieldSchema, StorageCollectionSchema } from './types.js';
 
+export function isNullable(field: StorageFieldSchema) {
+	if (field.type === 'any') return true;
+	if (field.type === 'map') return false;
+	return field.nullable;
+}
+
 export function addFieldDefaults(
 	collection: StorageCollectionSchema,
 	value: any,
 ) {
 	for (const [key, field] of Object.entries(collection.fields)) {
 		const defaultValue = getFieldDefault(field);
-		if (defaultValue !== undefined && value[key] === undefined) {
+		if (
+			(defaultValue !== undefined && value[key] === undefined) ||
+			// covers the case where a previously nullable field
+			// now has a default during a new migration
+			(!isNullable(field) && value[key] === null)
+		) {
 			value[key] = defaultValue;
 		}
 		if (value[key]) {
@@ -70,3 +82,45 @@ export function getFieldDefault(field: StorageFieldSchema) {
 	}
 	return undefined;
 }
+
+export function removeExtraProperties(
+	collection: StorageCollectionSchema,
+	value: any,
+) {
+	for (const [key, fieldValue] of Object.entries(value)) {
+		// MUST NOT DELETE THESE!
+		if (key === OID_KEY || key === LEGACY_OID_KEY) continue;
+
+		if (!collection.fields[key]) {
+			delete value[key];
+		} else {
+			traverseCollectionFieldsAndRemoveExtraProperties(
+				fieldValue,
+				collection.fields[key],
+			);
+		}
+	}
+	return value;
+}
+
+export function traverseCollectionFieldsAndRemoveExtraProperties(
+	value: any,
+	field: StorageFieldSchema,
+) {
+	if (isObject(value) && field.type === 'object') {
+		for (const [key, fieldValue] of Object.entries(value)) {
+			if (!field.properties[key]) {
+				delete value[key];
+			} else {
+				traverseCollectionFieldsAndRemoveExtraProperties(
+					fieldValue,
+					field.properties[key],
+				);
+			}
+		}
+	} else if (Array.isArray(value) && field.type === 'array') {
+		for (const item of value) {
+			traverseCollectionFieldsAndRemoveExtraProperties(item, field.items);
+		}
+	}
+}

package/src/schema/index.ts

@@ -25,4 +25,4 @@ export function schema<
 export * from './types.js';
 
 export * from './indexFilters.js';
-export * from './defaults.js';
+export * from './fields.js';