@verdant-web/common 2.8.0 → 2.9.1

package/src/diffing.ts

@@ -1,7 +1,9 @@
+import { AuthorizationKey } from './authz.js';
 import { VerdantError } from './error.js';
 import {
 	areOidsRelated,
 	assignOid,
+	createRef,
 	getOid,
 	maybeGetOid,
 	ObjectIdentifier,
@@ -27,7 +29,7 @@ export type DiffContext = {
 	/**
 	 * Authorization to apply to all created operations
 	 */
-	authz?: string;
+	authz?: AuthorizationKey;
 	patchCreator: PatchCreator;
 };
 
@@ -70,18 +72,10 @@ function enforceAssignedOid(
 			assignOid(newObject, existingObjectOid);
 		}
 		// NOTE: new OID assignments are done in patchCreator.
-		// else {
-		// 	assignOid(
-		// 		newObject,
-		// 		createSubOid(parentOid, ctx.patchCreator.createSubId),
-		// 	);
-		// }
 	} else if (!areOidsRelated(parentOid, oid)) {
 		// when there's any doubt, clone the whole object. false -> do not copy OIDs
 		const clone = cloneDeep(newObject, false);
 		// NOTE: new OID assignments are done in patchCreator.
-		// const cloneOid = createSubOid(parentOid, ctx.patchCreator.createSubId);
-		// assignOid(clone, cloneOid);
 		return clone;
 	}
 	return newObject;
@@ -102,7 +96,7 @@ export function diffToPatches(
 		merge?: boolean;
 		/** @deprecated - use 'merge' */
 		defaultUndefined?: boolean;
-		authz?: string;
+		authz?: AuthorizationKey;
 	},
 ) {
 	const ctx: DiffContext = {
@@ -141,7 +135,17 @@ export function diffLists(from: any[], to: any[], ctx: DiffContext) {
 	// from object must be registered with an OID.
 	const oid = getOid(from);
 	// this copy will be mutated to align with inserts, making things easier to compare.
-	const fromCopy = [...from];
+	const fromCopy = [];
+	// while iterating from, also collect some metadata.
+	const oidsInFrom = new Set();
+	for (let i = 0; i < from.length; i++) {
+		const value = from[i];
+		fromCopy[i] = value;
+		const oid = maybeGetOid(value);
+		if (oid) {
+			oidsInFrom.add(oid);
+		}
+	}
 
 	// first, normalize incoming data according to OID rules. this is done early
 	// so future equality checks are according to configuration like mergeUnknownObjects.
@@ -157,12 +161,17 @@ export function diffLists(from: any[], to: any[], ctx: DiffContext) {
 	// track whether the new list has gaps in it. a gap
 	// means we can no longer use list-push for new items.
 	let noGaps = true;
+	// track any items we move by reference, so we remember to
+	// not delete them later.
+	const movedItems = new Set<ObjectIdentifier>();
+	const overwrittenItems = new Set<ObjectIdentifier>();
 	for (let i = 0; i < to.length; i++) {
 		const value = to[i];
 		const oldValue = fromCopy[i];
 		if (value === undefined) {
 			noGaps = false;
 		}
+		const itemOid = maybeGetOid(value);
 
 		// we decide if this item is being added to the end of the list if:
 		// - there were no empty spaces before it in the new list
@@ -173,14 +182,49 @@ export function diffLists(from: any[], to: any[], ctx: DiffContext) {
 		// accidentally 'push' into a gap, which would put the item in the wrong place.
 		const isEndOfList = noGaps && i >= fromCopy.length;
 
-		if (isEndOfList) {
+		// this branch exclusively deals with object references.
+		if (itemOid && oidsInFrom.has(itemOid)) {
+			// this item was in the old list, so no matter what we want to
+			// preserve its identity. the rest of the branches here all
+			// assume the item is new and initialize it.
+			if (areTheSameIdentity(value, oldValue)) {
+				// the item hasn't moved. we just diff it in place.
+				diff(oldValue, value, ctx);
+			} else {
+				// the item has moved. we are currently visiting its new index,
+				// so we can move it by reference now
+				ctx.patches.push(
+					...ctx.patchCreator.createListMoveByRef(
+						oid,
+						createRef(itemOid),
+						i,
+						ctx.authz,
+					),
+				);
+				// the problem is, a move by ref is not the same as what
+				// we really observed here, since it's more like an insertion
+				// and not an overwrite. to complicate this, the replaced item
+				// at this index might actually have moved, not been replaced.
+				// we mark the item that was at this index previously, and if
+				// we don't see it later, we'll delete it.
+				const fromOid = maybeGetOid(oldValue);
+				if (fromOid) {
+					overwrittenItems.add(fromOid);
+				}
+
+				// remember this item so we don't delete it later.
+				movedItems.add(itemOid);
+			}
+		} else if (isEndOfList) {
 			// this will initialize all sub-objects in the item, too
 			ctx.patches.push(
 				...ctx.patchCreator.createListPush(oid, value, ctx.authz),
 			);
 		} else if (areTheSameIdentity(value, oldValue)) {
-			// the identity of this item hasn't changed, but we
-			// still have to diff the contents.
+			// note : since the branch above covers object references which
+			// were present in the original list, logically this branch only
+			// covers primitives. either way, call diff, it's just more
+			// proper to do so.
 			diff(oldValue, value, ctx);
 		} else {
 			// the identity of this item has changed. we can now evaluate
@@ -220,22 +264,55 @@ export function diffLists(from: any[], to: any[], ctx: DiffContext) {
 		}
 	}
 
-	// remove any remaining items at the end of the array
+	// remove any remaining items at the end of the array.
 	const deletedItemsAtEnd = fromCopy.length - to.length;
 	if (deletedItemsAtEnd > 0) {
+		for (let i = fromCopy.length - 1; i >= to.length; i--) {
+			const value = fromCopy[i];
+			const itemOid = maybeGetOid(value);
+			if (itemOid) {
+				// avoid deleting moved objects that were at the end of the list before.
+				// their reference was moved elsewhere, but their contents must be retained.
+				if (!movedItems.has(itemOid)) {
+					// if sub-items were objects, we need to delete them all
+					// this should recursively delete children of these items
+					// also!
+					deleteWithSubObjects(value, ctx);
+					ctx.patches.push(
+						...ctx.patchCreator.createListRemove(
+							oid,
+							createRef(itemOid),
+							'last',
+							ctx.authz,
+						),
+					);
+				}
+			} else {
+				ctx.patches.push(
+					...ctx.patchCreator.createListRemove(oid, value, 'last', ctx.authz),
+				);
+			}
+		}
+	}
+
+	for (const overwrittenOid of overwrittenItems) {
+		if (movedItems.has(overwrittenOid)) {
+			// this item was moved, not overwritten. we don't want to delete it.
+			continue;
+		}
+		// this item was overwritten, not moved. we need to delete it.
 		// if sub-items were objects, we need to delete them all
 		// this should recursively delete children of these items
 		// also!
-		for (let i = to.length; i < fromCopy.length; i++) {
-			const value = fromCopy[i];
-			deleteWithSubObjects(value, ctx);
+		const item = from.find((x) => maybeGetOid(x) === overwrittenOid);
+		if (item) {
+			deleteWithSubObjects(item, ctx);
 		}
-		// push the list-delete for the deleted items
 		ctx.patches.push(
-			...ctx.patchCreator.createListDelete(
+			...ctx.patchCreator.createListRemove(
 				oid,
-				to.length,
-				deletedItemsAtEnd,
+				createRef(overwrittenOid),
+				'last',
 				ctx.authz,
 			),
 		);

package/src/oids.ts

@@ -2,7 +2,7 @@ import { v4 } from 'uuid';
 import { isFileRef } from './files.js';
 import { isObjectRef, ObjectRef } from './operation.js';
 import { isRef } from './refs.js';
-import { isObject, assert } from './utils.js';
+import { assert, isObject } from './utils.js';
 
 /**
  * OIDs
@@ -421,3 +421,25 @@ function migrateForeignOid(parentOid: ObjectIdentifier, child: any) {
 		);
 	}
 }
+
+/**
+ * Returns a list of all OIDs assigned to this object
+ * and all sub-objects.
+ */
+export function getAllOids(root: any) {
+	const oids = new Set<ObjectIdentifier>();
+	const stack = [root];
+	while (stack.length) {
+		const obj = stack.pop();
+		const oid = maybeGetOid(obj);
+		if (oid) {
+			oids.add(oid);
+		}
+		if (Array.isArray(obj)) {
+			stack.push(...obj);
+		} else if (isObject(obj)) {
+			stack.push(...Object.values(obj));
+		}
+	}
+	return Array.from(oids);
+}

package/src/operation.ts

@@ -1,3 +1,4 @@
+import { AuthorizationKey } from './authz.js';
 import { DocumentBaseline } from './baseline.js';
 import { FileRef, isFileRef } from './files.js';
 import {
@@ -151,7 +152,7 @@ export type Operation = {
 	oid: ObjectIdentifier;
 	timestamp: string;
 	data: OperationPatch;
-	authz?: string;
+	authz?: AuthorizationKey;
 };
 
 /**
@@ -164,7 +165,7 @@ export function initialToPatches(
 	getNow: () => string,
 	createSubId?: () => string,
 	patches: Operation[] = [],
-	options?: { authz?: string },
+	options?: { authz?: AuthorizationKey },
 ) {
 	assignOid(initial, rootOid);
 	assignOidsToAllSubObjects(initial, createSubId);
@@ -189,7 +190,7 @@ export function shallowInitialToPatches(
 	rootOid: ObjectIdentifier,
 	getNow: () => string,
 	patches: Operation[] = [],
-	options?: { authz?: string },
+	options?: { authz?: AuthorizationKey },
 ) {
 	const op: Operation = {
 		oid: rootOid,
@@ -205,7 +206,7 @@ export function shallowInitialToPatches(
 
 // saves a bit of network traffic by not attaching authz
 // key at all if not present
-export function addAuthzToOp(op: Operation, authz?: string) {
+export function addAuthzToOp(op: Operation, authz?: AuthorizationKey) {
 	if (authz) {
 		op.authz = authz;
 	}
@@ -342,18 +343,18 @@ export function applyPatch<T extends NormalizedObject>(
 				do {
 					const valueToRemove = patch.value;
 					if (patch.only === 'last') {
-						if (isObjectRef(valueToRemove)) {
+						if (isRef(valueToRemove)) {
 							index = findLastIndex(
 								base,
-								(item: any) => item.id === valueToRemove.id,
+								(item: any) => isRef(item) && compareRefs(item, valueToRemove),
 							);
 						} else {
 							index = base.lastIndexOf(valueToRemove);
 						}
 					} else {
-						if (isObjectRef(valueToRemove)) {
+						if (isRef(valueToRemove)) {
 							index = base.findIndex(
-								(item: any) => item.id === valueToRemove.id,
+								(item: any) => isRef(item) && compareRefs(item, valueToRemove),
 							);
 						} else {
 							index = base.indexOf(valueToRemove);

package/src/patch.ts

@@ -11,9 +11,11 @@ import {
 	ObjectRef,
 	Operation,
 	PropertyName,
+	PropertyValue,
+	shallowInitialToPatches,
 } from './operation.js';
 import { isRef } from './refs.js';
-import { isObject } from './utils.js';
+import { assert, isObject } from './utils.js';
 
 export class PatchCreator {
 	constructor(
@@ -28,7 +30,13 @@ export class PatchCreator {
 	createDiff = (
 		from: any,
 		to: any,
-		options: { mergeUnknownObjects?: boolean; defaultUndefined?: boolean } = {},
+		options: {
+			mergeUnknownObjects?: boolean;
+			/** @deprecated use the equivalent 'merge' */
+			defaultUndefined?: boolean;
+			merge?: boolean;
+			authz?: AuthorizationKey;
+		} = {},
 	) => {
 		return diffToPatches(from, to, this.getNow, this.createSubId, [], options);
 	};
@@ -37,7 +45,17 @@ export class PatchCreator {
 		obj: any,
 		oid: ObjectIdentifier,
 		authz?: AuthorizationKey,
+		shallow?: boolean,
 	) => {
+		if (shallow) {
+			return shallowInitialToPatches(
+				obj,
+				oid,
+				this.getNow,
+				undefined,
+				authz ? { authz } : undefined,
+			);
+		}
 		return initialToPatches(
 			obj,
 			oid,
@@ -52,7 +70,7 @@ export class PatchCreator {
 		oid: ObjectIdentifier,
 		key: PropertyName,
 		value: any,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
 		// incoming value must be normalized. if it's not a primitive, it and all sub-objects
 		// must be created
@@ -102,7 +120,7 @@ export class PatchCreator {
 	createRemove = (
 		oid: ObjectIdentifier,
 		key: PropertyName,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
 		return [
 			{
@@ -121,8 +139,9 @@ export class PatchCreator {
 		oid: ObjectIdentifier,
 		index: number,
 		value: any,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
+		assert(index >= 0, 'List index must be non-negative');
 		if (this.isPrimitive(value)) {
 			return [
 				{
@@ -166,7 +185,7 @@ export class PatchCreator {
 	createListPush = (
 		oid: ObjectIdentifier,
 		value: any,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
 		if (this.isPrimitive(value)) {
 			return [
@@ -202,7 +221,7 @@ export class PatchCreator {
 	createListAdd = (
 		oid: ObjectIdentifier,
 		value: any,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
 		if (!this.isPrimitive(value)) {
 			return [
@@ -235,8 +254,9 @@ export class PatchCreator {
 		oid: ObjectIdentifier,
 		index: number,
 		value: any,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
+		assert(index >= 0, 'List index must be non-negative');
 		if (this.isPrimitive(value)) {
 			return [
 				{
@@ -253,9 +273,18 @@ export class PatchCreator {
 		} else {
 			const itemOid = createSubOid(oid, this.createSubId);
 			return [
-				...initialToPatches(value, itemOid, this.getNow, undefined, undefined, {
-					authz,
-				}),
+				...initialToPatches(
+					value,
+					itemOid,
+					this.getNow,
+					this.createSubId,
+					undefined,
+					authz
+						? {
+								authz,
+							}
+						: undefined,
+				),
 				{
 					oid,
 					timestamp: this.getNow(),
@@ -270,11 +299,49 @@ export class PatchCreator {
 		}
 	};
 
+	createListInsertMany = (
+		oid: ObjectIdentifier,
+		index: number,
+		values: any[],
+		authz?: AuthorizationKey,
+	): Operation[] => {
+		assert(index >= 0, 'List index must be non-negative');
+		const operations: Operation[] = [];
+		const refs = values.map((value) => {
+			if (this.isPrimitive(value)) return value;
+			const subOid = createSubOid(oid, this.createSubId);
+			operations.push(
+				...initialToPatches(
+					value,
+					subOid,
+					this.getNow,
+					this.createSubId,
+					undefined,
+					{
+						authz,
+					},
+				),
+			);
+			return createRef(subOid);
+		});
+		operations.push({
+			oid,
+			timestamp: this.getNow(),
+			data: {
+				op: 'list-insert',
+				values: refs,
+				index,
+			},
+			authz,
+		});
+		return operations;
+	};
+
 	createListRemove = (
 		oid: ObjectIdentifier,
-		value: any,
+		value: PropertyValue,
 		only?: 'first' | 'last',
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
 		return [
 			{
@@ -294,8 +361,10 @@ export class PatchCreator {
 		oid: ObjectIdentifier,
 		index: number,
 		count: number = 1,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
+		assert(index >= 0, 'List index must be non-negative');
+		assert(count > 0, 'Count must be positive and non-zero');
 		return [
 			{
 				oid,
@@ -314,8 +383,9 @@ export class PatchCreator {
 		oid: ObjectIdentifier,
 		value: ObjectRef | FileRef,
 		index: number,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
+		assert(index >= 0, 'List index must be non-negative');
 		return [
 			{
 				oid,
@@ -334,8 +404,10 @@ export class PatchCreator {
 		oid: ObjectIdentifier,
 		fromIndex: number,
 		toIndex: number,
-		authz?: string,
+		authz?: AuthorizationKey,
 	): Operation[] => {
+		assert(fromIndex >= 0, 'List move from index must be non-negative');
+		assert(toIndex >= 0, 'List move to index must be non-negative');
 		return [
 			{
 				oid,
@@ -350,7 +422,10 @@ export class PatchCreator {
 		];
 	};
 
-	createDelete = (oid: ObjectIdentifier, authz?: string): Operation[] => {
+	createDelete = (
+		oid: ObjectIdentifier,
+		authz?: AuthorizationKey,
+	): Operation[] => {
 		return [
 			{
 				oid,

package/src/schema/fields.ts

@@ -156,3 +156,11 @@ export function traverseCollectionFieldsAndRemoveExtraProperties(
 		}
 	}
 }
+
+export function isPrimitive(fieldSchema: StorageFieldSchema) {
+	return (
+		fieldSchema.type === 'string' ||
+		fieldSchema.type === 'number' ||
+		fieldSchema.type === 'boolean'
+	);
+}

package/src/schema/validation.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, it } from 'vitest';
+import { createFileRef } from '../files.js';
+import { createRef } from '../oids.js';
+import { schema, validateEntityField } from './index.js';
+
+describe('validation', () => {
+	it('validates shallowly with refs', () => {
+		const fieldSchema = schema.fields.object({
+			fields: {
+				obj: schema.fields.object({
+					fields: {},
+				}),
+				file: schema.fields.file(),
+			},
+		});
+		expect(
+			validateEntityField({
+				field: fieldSchema,
+				value: {
+					obj: createRef('some/id:1'),
+					file: createFileRef('somefile'),
+				},
+				fieldPath: [],
+				expectRefs: true,
+			}),
+		).toBeUndefined();
+		expect(
+			validateEntityField({
+				field: fieldSchema,
+				value: {
+					obj: createRef('some/id:1'),
+					file: createRef('some/object:1'),
+				},
+				fieldPath: [],
+				expectRefs: true,
+			}),
+		).toMatchInlineSnapshot(`
+			{
+			  "fieldPath": [
+			    "file",
+			  ],
+			  "message": "Expected file ref for field file, got {"@@type":"ref","id":"some/object:1"}",
+			  "type": "invalid-type",
+			}
+		`);
+		expect(
+			validateEntityField({
+				field: fieldSchema,
+				value: {
+					obj: 'foo',
+					file: createFileRef('somefile'),
+				},
+				fieldPath: [],
+				expectRefs: true,
+			}),
+		).toMatchInlineSnapshot(`
+			{
+			  "fieldPath": [
+			    "obj",
+			  ],
+			  "message": "Expected ref for field obj, got "foo"",
+			  "type": "invalid-type",
+			}
+		`);
+	});
+});