@verdant-web/common 1.12.0 → 1.13.1

package/src/migration.ts

@@ -6,6 +6,11 @@ import {
 	StorageDocument,
 	CollectionFilter,
 	StorageDocumentInit,
+	removeExtraProperties,
+	assert,
+	hasOid,
+	assignOid,
+	getOid,
 } from './index.js';
 
 export interface DroppedCollectionMigrationStrategy<
@@ -147,6 +152,12 @@ export interface MigrationTools<
 > {
 	migrate: MigrationRunner<Old, New>;
 	identity: <T>(val: T) => T;
+	/**
+	 * @deprecated - default field values are automatically
+	 * applied during migration, you don't need to use this.
+	 * Please remove it from your migrations - even old ones
+	 * (old migrations can be updated!)
+	 */
 	withDefaults: (collectionName: string, value: any) => any;
 	queries: MigrationQueries<Old>;
 	mutations: MigrationMutations<New>;
@@ -171,6 +182,7 @@ export interface MigrationEngine<
 	newOids: string[];
 	/** Promises that should be resolved before completing the migration */
 	awaitables: Promise<any>[];
+	log: (...messages: any[]) => void;
 }
 
 type MigrationProcedure<
@@ -226,6 +238,24 @@ export function migrate(
 	const addedCollections = Object.keys(newSchema.collections).filter(
 		(key) => !oldSchema.collections[key],
 	);
+	// collections which added one or more field defaults
+	const autoMigratedCollections = new Set<string>();
+	for (const collection of changedCollections) {
+		const oldFields = oldSchema.collections[collection].fields;
+		const newFields = newSchema.collections[collection].fields;
+		// a new default was added - we can auto-migrate it
+		if (
+			Object.keys(newFields).some(
+				(key) => !oldFields[key]?.default && newFields[key]?.default,
+			)
+		) {
+			autoMigratedCollections.add(collection);
+		}
+		// a field was removed - we can auto-migrate it
+		if (Object.keys(oldFields).some((key) => !newFields[key])) {
+			autoMigratedCollections.add(collection);
+		}
+	}
 
 	const addedIndexes: Record<string, MigrationIndexDescription[]> = {};
 	const removedIndexes: Record<string, MigrationIndexDescription[]> = {};
@@ -240,26 +270,47 @@ export function migrate(
 		);
 		if (added.length > 0) {
 			addedIndexes[changed] = added;
+			autoMigratedCollections.add(changed);
 		}
 		if (removed.length > 0) {
 			removedIndexes[changed] = removed;
+			autoMigratedCollections.add(changed);
 		}
 	}
 
+	const withDefaults = (collectionName: string, val: any) => {
+		return addFieldDefaults(newSchema.collections[collectionName], val);
+	};
+	const autoMigration = (collectionName: string) => (val: any) => {
+		const collection = newSchema.collections[collectionName];
+		return addFieldDefaults(collection, removeExtraProperties(collection, val));
+	};
+
 	return {
 		version: newSchema.version,
 		migrate: async (engine: MigrationEngine<any, any>) => {
 			const migratedCollections: string[] = [];
 			await procedure({
 				migrate: async (collection: any, strategy: any) => {
+					const auto = autoMigration(collection);
+					const wrapped = async (val: any) => {
+						const baseValue = await strategy(val);
+						// assign OID from original value in case user's strategy
+						// involves cloning
+						assignOid(baseValue, getOid(val));
+						const result = auto(baseValue);
+						return result;
+					};
 					// @ts-ignore
-					await engine.migrate(collection, strategy);
+					await engine.migrate(collection, wrapped);
 					migratedCollections.push(collection);
+					// since the user migrated this one and we wrap their
+					// strategy with auto-migration, we can remove it from
+					// the auto-migration list
+					autoMigratedCollections.delete(collection);
 				},
 				identity: (val: any) => val,
-				withDefaults: (collectionName: string, val: any) => {
-					return addFieldDefaults(newSchema.collections[collectionName], val);
-				},
+				withDefaults,
 				info: {
 					changedCollections,
 					addedCollections,
@@ -269,11 +320,30 @@ export function migrate(
 				mutations: engine.mutations,
 			});
 
-			const unmigrated = changedCollections.filter(
-				(collection) => !migratedCollections.includes(collection),
-			);
-			if (unmigrated.length > 0) {
-				console.error('Unmigrated changed collections:', unmigrated);
+			// mandatory migration of fields which had defaults added or
+			// fields removed but weren't migrated by the user
+
+			if (newSchema.version > 1) {
+				engine.log(
+					'debug',
+					'auto-migrating collections with new defaults',
+					autoMigratedCollections,
+				);
+				for (const name of autoMigratedCollections) {
+					await engine.migrate(name, autoMigration(name));
+					migratedCollections.push(name);
+				}
+
+				const unmigrated = changedCollections.filter(
+					(collection) => !migratedCollections.includes(collection),
+				);
+				if (unmigrated.length > 0) {
+					// TODO: does this deserve a full-on error?
+					console.error(
+						`Unmigrated changed collections from version ${oldSchema.version} to version ${newSchema.version}:`,
+						unmigrated,
+					);
+				}
 			}
 		},
 		removedCollections,
@@ -376,16 +446,12 @@ export function createDefaultMigration(
 				version: 0,
 				collections: {},
 		  };
-	return migrate(
-		oldSchema,
-		newSchema || schema,
-		async ({ migrate, withDefaults, info }) => {
-			if ((newSchema || schema).version === 1) return;
+	return migrate(oldSchema, newSchema || schema, async ({ migrate, info }) => {
+		if ((newSchema || schema).version === 1) return;
 
-			for (const collection of info.changedCollections as any) {
-				// @ts-ignore indefinite type resolution
-				await migrate(collection, (old) => withDefaults(collection, old));
-			}
-		},
-	);
+		for (const collection of info.changedCollections as any) {
+			// @ts-ignore indefinite type resolution
+			await migrate(collection, (old) => old);
+		}
+	});
 }

package/src/oids.test.ts

@@ -9,6 +9,7 @@ import {
 	decomposeOid,
 	getOid,
 	getOidRange,
+	getOidRoot,
 	hasOid,
 	maybeGetOidProperty,
 	normalize,
@@ -404,3 +405,9 @@ describe('assigning OIDs to sub-objects', () => {
 		expect(hasOid(obj.bar[0])).toBe(false);
 	});
 });
+
+it('should get the root OID for a legacy OID', () => [
+	expect(getOidRoot('items/clabgyjfh00003968qycsq3ld.inputs.#')).toEqual(
+		'items/clabgyjfh00003968qycsq3ld',
+	),
+]);

package/src/oids.ts

@@ -68,6 +68,9 @@ export function assignOid(obj: any, oid: ObjectIdentifier) {
 		isObject(obj),
 		`Only objects can be assigned OIDs, received ${JSON.stringify(obj)}`,
 	);
+	if (hasOid(obj)) {
+		removeOid(obj);
+	}
 	oidMap.set(obj, oid);
 	return obj;
 }
@@ -103,6 +106,27 @@ export function ensureOid(
 	}
 }
 
+export function ensureCompatibleOid(
+	obj: any,
+	rootOid: ObjectIdentifier,
+	createSubId?: () => string,
+) {
+	if (!hasOid(obj)) {
+		const oid = createSubOid(rootOid, createSubId);
+		assignOid(obj, oid);
+		return oid;
+	} else {
+		const existingOid = getOid(obj);
+		if (!areOidsRelated(existingOid, rootOid)) {
+			const oid = createSubOid(rootOid, createSubId);
+			assignOid(obj, oid);
+			return oid;
+		} else {
+			return getOid(obj);
+		}
+	}
+}
+
 const SANTIIZE_PLACEHOLDERS = {
 	'.': '˙',
 	'/': '&slash;',
@@ -188,14 +212,14 @@ export function assignOidsToAllSubObjects(
 		for (let i = 0; i < obj.length; i++) {
 			item = obj[i];
 			if (isObject(item) && !isRef(item)) {
-				ensureOid(item, rootOid, createSubId);
+				ensureCompatibleOid(item, rootOid, createSubId);
 				assignOidsToAllSubObjects(item, createSubId);
 			}
 		}
 	} else if (isObject(obj) && !isRef(obj)) {
 		for (const key of Object.keys(obj)) {
 			if (isObject(obj[key]) && !isRef(obj[key])) {
-				ensureOid(obj[key], rootOid, createSubId);
+				ensureCompatibleOid(obj[key], rootOid, createSubId);
 				assignOidsToAllSubObjects(obj[key], createSubId);
 			}
 		}
@@ -402,7 +426,7 @@ export function normalizeFirstLevel(obj: any) {
 }
 
 export function getOidRoot(oid: ObjectIdentifier) {
-	return oid.split(RANDOM_SEPARATOR)[0];
+	return oid.split('.')[0].split(RANDOM_SEPARATOR)[0];
 }
 
 /**
@@ -421,3 +445,7 @@ export function getRoots(oids: ObjectIdentifier[]) {
 	}
 	return Array.from(set);
 }
+
+export function areOidsRelated(oidA: ObjectIdentifier, oidB: ObjectIdentifier) {
+	return getOidRoot(oidA) === getOidRoot(oidB);
+}

package/src/operation.test.ts

@@ -18,7 +18,13 @@ import {
 
 function createClock(init = 0) {
 	let i = init;
-	return () => (i++).toString();
+	const clock = () => (i++).toString();
+	Object.defineProperty(clock, 'current', {
+		get() {
+			return i;
+		},
+	});
+	return clock;
 }
 
 describe('creating diff patch operations', () => {
@@ -509,6 +515,156 @@ describe('creating diff patch operations', () => {
 			]
 		`);
 	});
+
+	it('should assign a new OID to objects which had an incompatible existing OID', () => {
+		const from = {
+			foo: {
+				bar: [1, 2, 3],
+			},
+		};
+		assignOid(from, 'test/a');
+		assignOidsToAllSubObjects(from, createClock());
+		const to = {
+			foo: {
+				bar: [1, 2],
+			},
+		};
+		assignOid(to, 'test/a');
+		assignOid(to.foo, 'uff/b');
+		assignOidsToAllSubObjects(to.foo, createClock());
+		expect(
+			diffToPatches(from, to, createClock(), createClock(5), [], {
+				mergeUnknownObjects: true,
+			}),
+		).toMatchInlineSnapshot(`
+			[
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": [
+			        1,
+			        2,
+			      ],
+			    },
+			    "oid": "test/a:6",
+			    "timestamp": "0",
+			  },
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": {
+			        "bar": {
+			          "@@type": "ref",
+			          "id": "test/a:6",
+			        },
+			      },
+			    },
+			    "oid": "test/a:5",
+			    "timestamp": "1",
+			  },
+			  {
+			    "data": {
+			      "name": "foo",
+			      "op": "set",
+			      "value": {
+			        "@@type": "ref",
+			        "id": "test/a:5",
+			      },
+			    },
+			    "oid": "test/a",
+			    "timestamp": "2",
+			  },
+			  {
+			    "data": {
+			      "op": "delete",
+			    },
+			    "oid": "test/a:0",
+			    "timestamp": "3",
+			  },
+			]
+		`);
+	});
+
+	// an edge case - if a subobject which contains nested objects changes identity,
+	// this gets written as a new init patch tree - we want to ensure all init subobjects
+	// in that tree have compatible OIDs.
+	it('should assign a new OID to objects which have an incompatible OID when parent identity changes', () => {
+		const from = {
+			bar: [],
+		};
+		assignOid(from, 'test/a');
+		assignOid(from.bar, 'test/random');
+		const to = {
+			bar: [{ baz: 1 }, { baz: 2 }],
+		};
+		assignOid(to, 'test/a');
+		assignOidsToAllSubObjects(to, createClock());
+		assignOid(to.bar[0], 'uff/b');
+		expect(
+			diffToPatches(from, to, createClock(), createClock(5), [], {
+				mergeUnknownObjects: true,
+			}),
+		).toMatchInlineSnapshot(`
+			[
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": {
+			        "baz": 1,
+			      },
+			    },
+			    "oid": "test/a:5",
+			    "timestamp": "0",
+			  },
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": {
+			        "baz": 2,
+			      },
+			    },
+			    "oid": "test/a:2",
+			    "timestamp": "1",
+			  },
+			  {
+			    "data": {
+			      "op": "initialize",
+			      "value": [
+			        {
+			          "@@type": "ref",
+			          "id": "test/a:5",
+			        },
+			        {
+			          "@@type": "ref",
+			          "id": "test/a:2",
+			        },
+			      ],
+			    },
+			    "oid": "test/a:0",
+			    "timestamp": "2",
+			  },
+			  {
+			    "data": {
+			      "name": "bar",
+			      "op": "set",
+			      "value": {
+			        "@@type": "ref",
+			        "id": "test/a:0",
+			      },
+			    },
+			    "oid": "test/a",
+			    "timestamp": "3",
+			  },
+			  {
+			    "data": {
+			      "op": "delete",
+			    },
+			    "oid": "test/random",
+			    "timestamp": "4",
+			  },
+			]
+		`);
+	});
 });
 
 describe('substituting refs with objects', () => {

package/src/operation.ts

@@ -1,9 +1,11 @@
 import { DocumentBaseline } from './baseline.js';
 import { FileRef } from './files.js';
 import {
+	areOidsRelated,
 	assignOid,
 	assignOidsToAllSubObjects,
 	createRef,
+	createSubOid,
 	ensureOid,
 	getOid,
 	getOidRoot,
@@ -192,13 +194,19 @@ export function diffToPatches<T extends { [key: string]: any } | any[]>(
 			}
 		} else {
 			const oldValueOid = maybeGetOid(oldValue);
-			let valueOid;
-			if (!options.mergeUnknownObjects) {
+			let valueOid = maybeGetOid(value);
+			// the user supplied an object which was already assigned an OID,
+			// but that OID is from a separate entity: this object must
+			// be cloned to a new identity.
+			if (valueOid && !areOidsRelated(oid, valueOid)) {
+				value = cloneDeep(value, false);
+				valueOid = createSubOid(oid, createSubId);
+				assignOid(value, valueOid);
+			} else if (!options.mergeUnknownObjects) {
 				valueOid = ensureOid(value, oid, createSubId);
 			} else {
 				// if merge unknown objects is requested, we copy the previous value's oid
 				// to any mirrored new value if it doesn't have one assigned already.
-				valueOid = maybeGetOid(value);
 				if (!valueOid && oldValueOid) {
 					assignOid(value, oldValueOid);
 					valueOid = oldValueOid;
@@ -206,6 +214,14 @@ export function diffToPatches<T extends { [key: string]: any } | any[]>(
 					valueOid = ensureOid(value, oid, createSubId);
 				}
 			}
+			assert(
+				!!valueOid,
+				'Error: no value OID was resolved during diff. This is a bug in Verdant.',
+			);
+			assert(
+				areOidsRelated(oid, valueOid),
+				`Error: value OID ${valueOid} is not related to parent OID ${oid}. This is a bug in Verdant.`,
+			);
 
 			if (oldValue === undefined || valueOid !== oldValueOid) {
 				// first case: previous value exists but the OIDs are different,

package/src/schema/defaults.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { getFieldDefault } from './defaults.js';
+import { getFieldDefault } from './fields.js';
 import { assignOid, createOid, getOid, maybeGetOid } from '../oids.js';
 
 describe('getting field defaults', () => {

package/src/schema/{defaults.ts → fields.ts}

@@ -1,13 +1,25 @@
 import { LEGACY_OID_KEY, OID_KEY } from '../oids.js';
+import { isObject } from '../utils.js';
 import type { StorageFieldSchema, StorageCollectionSchema } from './types.js';
 
+export function isNullable(field: StorageFieldSchema) {
+	if (field.type === 'any') return true;
+	if (field.type === 'map') return false;
+	return field.nullable;
+}
+
 export function addFieldDefaults(
 	collection: StorageCollectionSchema,
 	value: any,
 ) {
 	for (const [key, field] of Object.entries(collection.fields)) {
 		const defaultValue = getFieldDefault(field);
-		if (defaultValue !== undefined && value[key] === undefined) {
+		if (
+			(defaultValue !== undefined && value[key] === undefined) ||
+			// covers the case where a previously nullable field
+			// now has a default during a new migration
+			(!isNullable(field) && value[key] === null)
+		) {
 			value[key] = defaultValue;
 		}
 		if (value[key]) {
@@ -70,3 +82,45 @@ export function getFieldDefault(field: StorageFieldSchema) {
 	}
 	return undefined;
 }
+
+export function removeExtraProperties(
+	collection: StorageCollectionSchema,
+	value: any,
+) {
+	for (const [key, fieldValue] of Object.entries(value)) {
+		// MUST NOT DELETE THESE!
+		if (key === OID_KEY || key === LEGACY_OID_KEY) continue;
+
+		if (!collection.fields[key]) {
+			delete value[key];
+		} else {
+			traverseCollectionFieldsAndRemoveExtraProperties(
+				fieldValue,
+				collection.fields[key],
+			);
+		}
+	}
+	return value;
+}
+
+export function traverseCollectionFieldsAndRemoveExtraProperties(
+	value: any,
+	field: StorageFieldSchema,
+) {
+	if (isObject(value) && field.type === 'object') {
+		for (const [key, fieldValue] of Object.entries(value)) {
+			if (!field.properties[key]) {
+				delete value[key];
+			} else {
+				traverseCollectionFieldsAndRemoveExtraProperties(
+					fieldValue,
+					field.properties[key],
+				);
+			}
+		}
+	} else if (Array.isArray(value) && field.type === 'array') {
+		for (const item of value) {
+			traverseCollectionFieldsAndRemoveExtraProperties(item, field.items);
+		}
+	}
+}

package/src/schema/index.ts

@@ -25,4 +25,4 @@ export function schema<
 export * from './types.js';
 
 export * from './indexFilters.js';
-export * from './defaults.js';
+export * from './fields.js';

package/src/schema/types/fields.ts

@@ -37,7 +37,12 @@ export type StorageMapFieldSchema<V extends NestedStorageFieldSchema> = {
 export type StorageFileFieldSchema = {
 	type: 'file';
 	nullable?: boolean;
-	// downloadRemote?: boolean; TODO: is this a good feature? maybe later.
+	/**
+	 * Instructs the client to download synced files to local storage on first request for offline use.
+	 * Leave this false to save storage space on the client, at the cost of requiring a network
+	 * connection to use files created by other devices.
+	 */
+	downloadRemote?: boolean;
 };
 
 export type StorageFieldSchema =

package/src/schema/validation.ts

@@ -0,0 +1,99 @@
+import { LEGACY_OID_KEY, OID_KEY } from '../oids.js';
+import { isObject } from '../utils.js';
+import { StorageCollectionSchema, StorageFieldSchema } from './types.js';
+
+export function validateEntity(schema: StorageCollectionSchema, entity: any) {
+	for (const [key, value] of Object.entries(entity)) {
+		if (!schema.fields[key]) {
+			throw new Error(`Invalid field "${key}"`);
+		}
+		if (value) {
+			validateEntityField(schema.fields[key], value);
+		}
+	}
+}
+
+export function validateEntityField(
+	field: StorageFieldSchema,
+	value: any,
+	fieldPath: string[] = [],
+) {
+	if (field.type === 'object') {
+		if (!isObject(value)) {
+			if (value === null && field.nullable) return;
+			throw new Error(
+				`Expected object${
+					field.nullable ? ' or null' : ''
+				} for field ${formatField(fieldPath)}}`,
+			);
+		}
+		for (const [key, subField] of Object.entries(field.properties)) {
+			if (value[key]) {
+				validateEntityField(subField, value[key]);
+			}
+		}
+		for (const key of Object.keys(value)) {
+			if (!field.properties[key]) {
+				throw new Error(
+					`Invalid field "${key}" on value ${formatField(fieldPath)}`,
+				);
+			}
+		}
+	} else if (field.type === 'array') {
+		if (!Array.isArray(value)) {
+			if (value === null && field.nullable) return;
+			throw new Error(
+				`Expected array${
+					field.nullable ? ' or null' : ''
+				} for field ${formatField(fieldPath)}}`,
+			);
+		}
+		for (const item of value) {
+			validateEntityField(field.items, item);
+		}
+	} else if (field.type === 'map') {
+		if (!isObject(value)) {
+			throw new Error(`Expected map for field ${formatField(fieldPath)}}`);
+		}
+		for (const [key, item] of Object.entries(value)) {
+			// santiy check to weed out any id field
+			if (key === OID_KEY || key === LEGACY_OID_KEY) continue;
+			validateEntityField(field.values, item);
+		}
+	} else if (field.type === 'string') {
+		if (typeof value !== 'string') {
+			if (value === null && field.nullable) return;
+			throw new Error(
+				`Expected string ${
+					field.nullable ? ' or null' : ''
+				} for field ${formatField(fieldPath)}}`,
+			);
+		}
+	} else if (field.type === 'boolean') {
+		if (typeof value !== 'boolean') {
+			if (value === null && field.nullable) return;
+			throw new Error(
+				`Expected boolean ${
+					field.nullable ? ' or null' : ''
+				} for field ${formatField(fieldPath)}}`,
+			);
+		}
+	} else if (field.type === 'number') {
+		if (typeof value !== 'number') {
+			if (value === null && field.nullable) return;
+			throw new Error(
+				`Expected number ${
+					field.nullable ? ' or null' : ''
+				} for field ${formatField(fieldPath)}}`,
+			);
+		}
+	} else if (field.type === 'file') {
+		if (value === null && !field.nullable) {
+			throw new Error(`Expected file for field ${formatField(fieldPath)}}`);
+		}
+	}
+}
+
+function formatField(fieldPath: string[]) {
+	return fieldPath.join('.');
+}