@verdant-web/common 1.14.1 → 1.15.0

package/src/migration.ts

@@ -11,29 +11,32 @@ import {
 	hasOid,
 	assignOid,
 	getOid,
+	hasDefault,
 } from './index.js';
 
+/**@deprecated */
 export interface DroppedCollectionMigrationStrategy<
 	Old extends StorageCollectionSchema<any, any, any>,
 > {
 	(old: Old): void | Promise<void>;
 }
+/**@deprecated */
 export interface PreservedCollectionMigrationStrategy<
 	Old extends StorageCollectionSchema<any, any, any>,
 	New extends StorageCollectionSchema<any, any, any>,
 > {
-	(old: StorageDocument<Old>):
-		| StorageDocument<New>
-		| Promise<StorageDocument<New>>;
+	(
+		old: StorageDocument<Old>,
+	): StorageDocument<New> | Promise<StorageDocument<New>>;
 }
-
+/** @deprecated */
 type MigrationStrategy<
 	Old extends StorageCollectionSchema<any, any, any>,
 	New extends StorageCollectionSchema<any, any, any>,
 > =
 	| DroppedCollectionMigrationStrategy<Old>
 	| PreservedCollectionMigrationStrategy<Old, New>;
-
+/** @deprecated */
 export type MigrationsKeyedOnCollection<
 	Old extends StorageSchema<any>,
 	New extends StorageSchema<any>,
@@ -41,25 +44,17 @@ export type MigrationsKeyedOnCollection<
 	| PreservedCollectionMigrations<Old, New>
 	| DroppedCollectionMigrations<Old, New>;
 
+/**@deprecated */
 type NotInSchema<
 	Name extends string | number | symbol,
 	Schema extends StorageSchema<any>,
 > = Name extends keyof Schema['collections'] ? never : Name;
+/** @deprecated */
 type InSchema<
 	Name extends string | number | symbol,
 	Schema extends StorageSchema<any>,
 > = Name extends keyof Schema['collections'] ? Name : never;
-
-type NewCollections<
-	Old extends StorageSchema<any>,
-	New extends StorageSchema<any>,
-> = {
-	[Key in keyof New['collections'] as NotInSchema<
-		Key,
-		Old
-	>]: StorageCollectionSchema<any, any, any>;
-};
-
+/** @deprecated */
 type DroppedCollections<
 	Old extends StorageSchema<any>,
 	New extends StorageSchema<any>,
@@ -69,7 +64,7 @@ type DroppedCollections<
 		New
 	>]: StorageCollectionSchema<any, any, any>;
 };
-
+/** @deprecated */
 type PreservedCollections<
 	Old extends StorageSchema<any>,
 	New extends StorageSchema<any>,
@@ -79,7 +74,7 @@ type PreservedCollections<
 		New
 	>]: StorageCollectionSchema<any, any, any>;
 };
-
+/** @deprecated */
 type DroppedCollectionMigrations<
 	Old extends StorageSchema<any>,
 	New extends StorageSchema<any>,
@@ -89,6 +84,7 @@ type DroppedCollectionMigrations<
 		New
 	>]: DroppedCollectionMigrationStrategy<Old['collections'][Key]>;
 };
+/** @deprecated */
 type PreservedCollectionMigrations<
 	Old extends StorageSchema<any>,
 	New extends StorageSchema<any>,
@@ -102,6 +98,7 @@ type PreservedCollectionMigrations<
 	>;
 };
 
+/** @deprecated */
 type StrategyFor<
 	Key extends string,
 	Old extends StorageSchema<any>,
@@ -113,7 +110,8 @@ type StrategyFor<
 	  >
 	: DroppedCollectionMigrationStrategy<Old['collections'][Key]>;
 
-type MigrationRunner<
+/** @deprecated */
+type DeprecatedMigrationRunner<
 	Old extends StorageSchema<any>,
 	New extends StorageSchema<any>,
 > = <Collection extends Extract<keyof Old['collections'], string>>(
@@ -121,7 +119,8 @@ type MigrationRunner<
 	strategy: StrategyFor<Collection, Old, New>,
 ) => Promise<void>;
 
-type MigrationQueryMaker<
+/** @deprecated */
+type DeprecatedMigrationQueryMaker<
 	Collection extends StorageCollectionSchema<any, any, any>,
 > = {
 	get(primaryKey: string): Promise<StorageDocument<Collection> | undefined>;
@@ -131,13 +130,15 @@ type MigrationQueryMaker<
 	findAll(query?: CollectionFilter): Promise<StorageDocument<Collection>[]>;
 };
 
-type MigrationQueries<Old extends StorageSchema<any>> = {
-	[Key in keyof Old['collections']]: MigrationQueryMaker<
+/** @deprecated */
+type DeprecatedMigrationQueries<Old extends StorageSchema<any>> = {
+	[Key in keyof Old['collections']]: DeprecatedMigrationQueryMaker<
 		Old['collections'][Key]
 	>;
 };
 
-type MigrationMutations<New extends StorageSchema> = {
+/** @deprecated */
+type DeprecatedMigrationMutations<New extends StorageSchema> = {
 	[Key in keyof New['collections']]: {
 		put(
 			document: StorageDocumentInit<New['collections'][Key]>,
@@ -145,12 +146,12 @@ type MigrationMutations<New extends StorageSchema> = {
 		delete(primaryKey: string): Promise<void>;
 	};
 };
-
-export interface MigrationTools<
+/** @deprecated */
+export interface DeprecatedMigrationTools<
 	Old extends StorageSchema<any>,
 	New extends StorageSchema<any>,
 > {
-	migrate: MigrationRunner<Old, New>;
+	migrate: DeprecatedMigrationRunner<Old, New>;
 	identity: <T>(val: T) => T;
 	/**
 	 * @deprecated - default field values are automatically
@@ -159,8 +160,8 @@ export interface MigrationTools<
 	 * (old migrations can be updated!)
 	 */
 	withDefaults: (collectionName: string, value: any) => any;
-	queries: MigrationQueries<Old>;
-	mutations: MigrationMutations<New>;
+	queries: DeprecatedMigrationQueries<Old>;
+	mutations: DeprecatedMigrationMutations<New>;
 	info: {
 		changedCollections: keyof Old['collections'][];
 		addedCollections: keyof New['collections'][];
@@ -168,27 +169,21 @@ export interface MigrationTools<
 	};
 }
 
-export interface MigrationEngine<
-	Old extends StorageSchema,
-	New extends StorageSchema,
-> {
-	migrate: (
-		collection: string,
-		strategy: MigrationStrategy<any, any>,
-	) => Promise<void>;
-	queries: MigrationQueries<Old>;
-	mutations: MigrationMutations<New>;
+export interface MigrationEngine {
+	migrate: (collection: string, strategy: (val: any) => any) => Promise<void>;
+	queries: MigrationQueries<any>;
+	mutations: MigrationMutations<any>;
 	/** OIDs of any new documents created during the migration */
 	newOids: string[];
 	/** Promises that should be resolved before completing the migration */
 	awaitables: Promise<any>[];
 	log: (...messages: any[]) => void;
 }
-
-type MigrationProcedure<
+/** @deprecated */
+type DeprecatedMigrationProcedure<
 	Old extends StorageSchema,
 	New extends StorageSchema,
-> = (tools: MigrationTools<Old, New>) => Promise<void>;
+> = (tools: DeprecatedMigrationTools<Old, New>) => Promise<void>;
 
 type EmptySchema = {
 	version: 0;
@@ -199,14 +194,16 @@ const emptySchema: EmptySchema = {
 	collections: {},
 };
 
+/** @deprecated - use createMigration */
 export function migrate<Schema extends StorageSchema>(
 	schema: Schema,
-	procedure: MigrationProcedure<EmptySchema, Schema>,
+	procedure: DeprecatedMigrationProcedure<EmptySchema, Schema>,
 ): Migration<EmptySchema, Schema>;
+/** @deprecated = use createMigration */
 export function migrate<Old extends StorageSchema, New extends StorageSchema>(
 	oldSchema: Old,
 	newSchema: New,
-	procedure: MigrationProcedure<Old, New>,
+	procedure: DeprecatedMigrationProcedure<Old, New>,
 ): Migration<Old, New>;
 export function migrate(
 	oldSchemaOrNewSchema: any,
@@ -294,7 +291,7 @@ export function migrate(
 
 	return {
 		version: newSchema.version,
-		migrate: async (engine: MigrationEngine<any, any>) => {
+		migrate: async (engine: MigrationEngine) => {
 			const migratedCollections: string[] = [];
 			await procedure({
 				migrate: async (collection: any, strategy: any) => {
@@ -378,7 +375,7 @@ export interface Migration<
 	version: number;
 	oldSchema: Old;
 	newSchema: New;
-	migrate: (engine: MigrationEngine<Old, New>) => Promise<void>;
+	migrate: (engine: MigrationEngine) => Promise<void>;
 	/** Collections which are added in the new schema and not present in the old */
 	addedCollections: string[];
 	/** Collections which were removed from the old schema */
@@ -403,21 +400,12 @@ function getIndexes<Coll extends StorageCollectionSchema<any, any, any>>(
 	collection: Coll | undefined,
 ): MigrationIndexDescription[] {
 	if (!collection) return [];
-	const fields = Object.keys(collection.fields)
-		.filter((key) => collection.fields[key].indexed)
-		.map((key) => ({
-			name: key,
-			multiEntry: collection.fields[key].type === 'array',
-			synthetic: false,
-			compound: false,
-		}));
 
 	return [
-		...fields,
-		...Object.keys(collection.synthetics || {}).map((key) => ({
+		...Object.keys(collection.indexes || {}).map((key) => ({
 			name: key,
 			multiEntry: ['array', 'string[]', 'number[]', 'boolean[]'].includes(
-				collection.synthetics[key].type,
+				collection.indexes[key].type,
 			),
 			synthetic: true,
 			compound: false,
@@ -426,7 +414,7 @@ function getIndexes<Coll extends StorageCollectionSchema<any, any, any>>(
 			name: key,
 			multiEntry: collection.compounds[key].of.some(
 				(fieldName: string) =>
-					(collection.fields[fieldName] || collection.synthetics[fieldName])
+					(collection.fields[fieldName] || collection.indexes[fieldName])
 						.type === 'array',
 			),
 			synthetic: false,
@@ -435,9 +423,11 @@ function getIndexes<Coll extends StorageCollectionSchema<any, any, any>>(
 	];
 }
 
+/** @deprecated - use createMigration with no procedure function */
 export function createDefaultMigration(
 	schema: StorageSchema,
 ): Migration<{ version: 0; collections: {} }>;
+/** @deprecated - use createMigration with no procedure function */
 export function createDefaultMigration<Old extends StorageSchema>(
 	oldSchema: Old,
 	newSchema: StorageSchema,
@@ -461,3 +451,256 @@ export function createDefaultMigration(
 		}
 	});
 }
+
+/** New, simpler type-safety migration tools */
+type DocumentShape<Init = any, Snapshot = any> = {
+	init: Init;
+	snapshot: Snapshot;
+};
+type SchemaDocuments = Record<string, DocumentShape>;
+type CollectionProcedure<OldSnapshot, NewInit> = {
+	(old: OldSnapshot): NewInit | Promise<NewInit>;
+};
+type MigrationQueries<Old extends SchemaDocuments> = {
+	[Key in keyof Old]: {
+		get(primaryKey: string): Promise<Old[Key]['snapshot'] | undefined>;
+		findOne(query: CollectionFilter): Promise<Old[Key]['snapshot'] | undefined>;
+		findAll(query?: CollectionFilter): Promise<Old[Key]['snapshot'][]>;
+	};
+};
+type MigrationMutations<New extends SchemaDocuments> = {
+	[Key in keyof New]: {
+		put(document: New[Key]['init']): Promise<New[Key]['snapshot']>;
+		delete(primaryKey: string): Promise<void>;
+	};
+};
+type MigrationTools<
+	Old extends SchemaDocuments,
+	New extends SchemaDocuments,
+> = {
+	/**
+	 * Process a change in a collection's documents by taking in each existing
+	 * document and returning its new shape. This is typed so you can be
+	 * confident the proper transformations are made.
+	 */
+	migrate: <Collection extends keyof Old & keyof New>(
+		collection: Collection,
+		strategy: CollectionProcedure<
+			Old[Collection]['snapshot'],
+			New[Collection]['init']
+		>,
+	) => Promise<void>;
+	mutations: MigrationMutations<New>;
+	queries: MigrationQueries<Old>;
+	info: {
+		changedCollections: (keyof Old & keyof New)[];
+		addedCollections: (keyof New)[];
+		removedCollections: (keyof Old)[];
+	};
+};
+type MigrationProcedure<
+	Old extends SchemaDocuments,
+	New extends SchemaDocuments,
+> = {
+	(tools: MigrationTools<Old, New>): Promise<void>;
+};
+type InitialMigrationTools<New extends SchemaDocuments> = {
+	mutations: MigrationMutations<New>;
+};
+type InitialMigrationProcedure<New extends SchemaDocuments> = {
+	(tools: InitialMigrationTools<New>): Promise<void>;
+};
+
+export function createMigration<New extends SchemaDocuments>(
+	newSchema: StorageSchema,
+	procedure?: InitialMigrationProcedure<New>,
+): any;
+export function createMigration<
+	Old extends SchemaDocuments,
+	New extends SchemaDocuments,
+>(
+	oldSchema: StorageSchema,
+	newSchema: StorageSchema,
+	procedure?: MigrationProcedure<Old, New>,
+): any;
+export function createMigration(
+	maybeOldSchema: StorageSchema,
+	maybeNewSchemaOrProcedure?:
+		| StorageSchema
+		| InitialMigrationProcedure<SchemaDocuments>,
+	maybeProcedure?: MigrationProcedure<SchemaDocuments, SchemaDocuments>,
+): any {
+	const isProcedureSecondArgument =
+		typeof maybeNewSchemaOrProcedure === 'function' ||
+		maybeNewSchemaOrProcedure === undefined;
+	const oldSchema = isProcedureSecondArgument ? emptySchema : maybeOldSchema;
+	const newSchema = isProcedureSecondArgument
+		? maybeOldSchema
+		: maybeNewSchemaOrProcedure;
+	const procedure = isProcedureSecondArgument
+		? maybeNewSchemaOrProcedure
+		: maybeProcedure;
+	assert(oldSchema, 'Invalid arguments to createMigration');
+	assert(newSchema, 'Invalid arguments to createMigration');
+	const {
+		changedCollections,
+		addedCollections,
+		removedCollections,
+		addedIndexes,
+		removedIndexes,
+		autoMigratedCollections,
+		autoMigration,
+	} = getMigrationInfo(oldSchema, newSchema);
+
+	return {
+		version: newSchema.version,
+		migrate: async (engine: MigrationEngine) => {
+			const migratedCollections: string[] = [];
+			const migrate = async (collection: any, strategy: any) => {
+				const auto = autoMigration(collection);
+				const wrapped = async (val: any) => {
+					const baseValue = await strategy(val);
+					// assign OID from original value in case user's strategy
+					// involves cloning
+					assignOid(baseValue, getOid(val));
+					const result = auto(baseValue);
+					return result;
+				};
+				// @ts-ignore
+				await engine.migrate(collection, wrapped);
+				migratedCollections.push(collection);
+				// since the user migrated this one and we wrap their
+				// strategy with auto-migration, we can remove it from
+				// the auto-migration list
+				autoMigratedCollections.delete(collection);
+			};
+			await procedure?.({
+				migrate,
+				info: {
+					changedCollections,
+					addedCollections,
+					removedCollections,
+				},
+				queries: engine.queries,
+				mutations: engine.mutations,
+			});
+
+			// mandatory migration of fields which had defaults added or
+			// fields removed but weren't migrated by the user
+
+			if (newSchema.version > 1) {
+				engine.log(
+					'debug',
+					'auto-migrating collections with new defaults',
+					autoMigratedCollections,
+				);
+				for (const name of autoMigratedCollections) {
+					await engine.migrate(name, autoMigration(name));
+					migratedCollections.push(name);
+				}
+
+				const unmigrated = changedCollections.filter(
+					(collection) => !migratedCollections.includes(collection),
+				);
+				if (unmigrated.length > 0) {
+					// TODO: does this deserve a full-on error?
+					console.error(
+						`Unmigrated changed collections from version ${oldSchema.version} to version ${newSchema.version}:`,
+						unmigrated,
+					);
+				}
+			}
+		},
+		removedCollections,
+		addedIndexes,
+		removedIndexes,
+		allCollections: Object.keys(newSchema.collections),
+		changedCollections,
+		addedCollections,
+		oldCollections: Object.keys(oldSchema.collections),
+		oldSchema,
+		newSchema,
+	};
+}
+
+// common tools
+function getMigrationInfo(oldSchema: StorageSchema, newSchema: StorageSchema) {
+	const changedCollections: string[] = Object.keys(
+		newSchema.collections,
+	).filter(
+		(key) =>
+			oldSchema.collections[key] &&
+			stableStringify(oldSchema.collections[key]) !==
+				stableStringify(newSchema.collections[key]),
+	);
+	const removedCollections: string[] = Object.keys(
+		oldSchema.collections,
+	).filter((key) => !newSchema.collections[key]);
+	const addedCollections = Object.keys(newSchema.collections).filter(
+		(key) => !oldSchema.collections[key],
+	);
+	// collections which added one or more field defaults
+	const autoMigratedCollections = new Set<string>();
+	for (const collection of changedCollections) {
+		const oldFields = oldSchema.collections[collection].fields;
+		const newFields = newSchema.collections[collection].fields;
+		// a new default was added - we can auto-migrate it
+		if (
+			Object.keys(newFields).some(
+				(key) => !hasDefault(oldFields[key]) && hasDefault(newFields[key]),
+			)
+		) {
+			autoMigratedCollections.add(collection);
+		}
+		// a field was removed - we can auto-migrate it
+		if (Object.keys(oldFields).some((key) => !newFields[key])) {
+			autoMigratedCollections.add(collection);
+		}
+	}
+
+	const addedIndexes: Record<string, MigrationIndexDescription[]> = {};
+	const removedIndexes: Record<string, MigrationIndexDescription[]> = {};
+	for (const changed of [...changedCollections, ...addedCollections]) {
+		const oldIndexes = getIndexes(oldSchema.collections[changed]);
+		const newIndexes = getIndexes(newSchema.collections[changed]);
+		const added = newIndexes.filter(
+			(index) => !oldIndexes.find((i) => i.name === index.name),
+		);
+		const removed = oldIndexes.filter(
+			(index) => !newIndexes.find((i) => i.name === index.name),
+		);
+		if (added.length > 0) {
+			addedIndexes[changed] = added;
+			// FIXME: don't o(n^2) this
+			if (changedCollections.includes(changed)) {
+				autoMigratedCollections.add(changed);
+			}
+		}
+		if (removed.length > 0) {
+			removedIndexes[changed] = removed;
+			// FIXME: don't o(n^2) this
+			if (changedCollections.includes(changed)) {
+				autoMigratedCollections.add(changed);
+			}
+		}
+	}
+
+	const withDefaults = (collectionName: string, val: any) => {
+		return addFieldDefaults(newSchema.collections[collectionName], val);
+	};
+	const autoMigration = (collectionName: string) => (val: any) => {
+		const collection = newSchema.collections[collectionName];
+		return addFieldDefaults(collection, removeExtraProperties(collection, val));
+	};
+
+	return {
+		changedCollections,
+		addedCollections,
+		removedCollections,
+		addedIndexes,
+		removedIndexes,
+		autoMigratedCollections,
+		withDefaults,
+		autoMigration,
+	};
+}

package/src/oids.test.ts

@@ -10,7 +10,7 @@ import {
 	createSubOid,
 	decomposeOid,
 	getOid,
-	getOidRange,
+	getOidSubIdRange,
 	getOidRoot,
 	hasOid,
 	maybeGetOidProperty,
@@ -262,11 +262,10 @@ describe('computing a range of oids for a whole object set', () => {
 		return oid >= start && oid <= end;
 	}
 	it('should include the root object and any possible sub object oid', () => {
-		const [start, end] = getOidRange('test/a');
-		expect(start).toEqual('test/a');
+		const [start, end] = getOidSubIdRange('test/a');
+		expect(start).toEqual('test/a:');
 		expect(end).toEqual('test/a:\uffff');
 		expect(start < end).toBe(true);
-		expect(isWithin('test/a', start, end)).toBe(true);
 		expect(isWithin('test/a:0', start, end)).toBe(true);
 		expect(isWithin('test/a:1', start, end)).toBe(true);
 		expect(isWithin('test/a:zzzzzzzzzzzzzzzzzzzzzzz', start, end)).toBe(true);
@@ -274,6 +273,8 @@ describe('computing a range of oids for a whole object set', () => {
 		expect(isWithin('test1/a', start, end)).toBe(false);
 		expect(isWithin('test/b', start, end)).toBe(false);
 		expect(isWithin('test/ ', start, end)).toBe(false);
+		expect(isWithin('test/a1', start, end)).toBe(false);
+		expect(isWithin('test/a1:3', start, end)).toBe(false);
 	});
 });
 

package/src/oids.ts

@@ -437,11 +437,12 @@ export function getOidRoot(oid: ObjectIdentifier) {
 
 /**
  * Returns an inclusive range of OIDs that represent
- * a root object and all of its sub-objects.
+ * all of an OID's possible sub-objects.
  */
-export function getOidRange(oid: ObjectIdentifier) {
+export function getOidSubIdRange(oid: ObjectIdentifier) {
 	const root = getOidRoot(oid);
-	return [root, `${root}${RANDOM_SEPARATOR}\uffff`];
+	const lastSubId = createSubOid(root, () => '\uffff');
+	return [`${root}${RANDOM_SEPARATOR}`, lastSubId];
 }
 
 export function getRoots(oids: ObjectIdentifier[]) {

package/src/schema/fields.ts

@@ -8,6 +8,24 @@ export function isNullable(field: StorageFieldSchema) {
 	return field.nullable;
 }
 
+export function hasDefault(field: StorageFieldSchema | undefined) {
+	if (!field) return false;
+	if (field.type === 'map') return true;
+	if (field.type === 'array') return true;
+	if (field.type === 'file') return false;
+	if (field.type === 'object') return true;
+	return field.default !== undefined;
+}
+
+export function isIndexed(field: StorageFieldSchema) {
+	if (field.type === 'map') return false;
+	if (field.type === 'object') return false;
+	if (field.type === 'array') return false;
+	if (field.type === 'file') return false;
+	if (field.type === 'any') return false;
+	return field.indexed;
+}
+
 export function addFieldDefaults(
 	collection: StorageCollectionSchema,
 	value: any,

package/src/schema/index.ts

@@ -1,8 +1,11 @@
+import { isIndexed } from './fields.js';
 import {
 	CollectionCompoundIndices,
 	StorageCollectionSchema,
+	StorageDirectSyntheticSchema,
 	StorageFieldsSchema,
 	StorageSchema,
+	StorageSyntheticIndexSchema,
 	StorageSyntheticIndices,
 } from './types.js';
 
@@ -10,15 +13,39 @@ export function collection<
 	Fields extends StorageFieldsSchema,
 	Synthetics extends StorageSyntheticIndices<Fields>,
 	Compounds extends CollectionCompoundIndices<Fields, Synthetics>,
->(input: StorageCollectionSchema<Fields, Synthetics, Compounds>) {
-	return input;
+>({
+	synthetics,
+	indexes,
+	...input
+}: StorageCollectionSchema<
+	Fields,
+	Synthetics,
+	Compounds
+>): StorageCollectionSchema<Fields, Synthetics, Compounds> {
+	// back compat - copy synthetics in with indexes
+	const finalIndexes = { ...synthetics, ...indexes };
+	// add all indexed fields into the synthetic indices (back compat)
+	for (const [key, field] of Object.entries(input.fields)) {
+		if (isIndexed(field)) {
+			finalIndexes[key] = {
+				field: key,
+			} as StorageDirectSyntheticSchema<Fields>;
+		}
+	}
+	return {
+		...input,
+		indexes: finalIndexes as Synthetics,
+	};
 }
 
 export function schema<
+	// Fields extends StorageFieldsSchema,
+	// Indexes extends StorageSyntheticIndices<Fields>,
+	// Compounds extends CollectionCompoundIndices<Fields, Indexes>,
 	Schema extends StorageSchema<{
 		[key: string]: StorageCollectionSchema<any, any, any>;
 	}>,
->(input: Schema) {
+>(input: Schema): StorageSchema {
 	return input;
 }