@contember/schema-migrations 1.2.0-alpha.18 → 1.2.0-alpha.19

package/src/MigrationFilesManager.ts

@@ -1,38 +1,28 @@
 import { MigrationVersionHelper } from './MigrationVersionHelper'
-import * as fs from 'fs'
-import { promisify } from 'util'
-import * as path from 'path'
-
-const readFile = promisify(fs.readFile)
-const fsWrite = promisify(fs.writeFile)
-const fsRemove = promisify(fs.unlink)
-const fsRealpath = promisify(fs.realpath)
-const mkdir = promisify(fs.mkdir)
-const lstatFile = promisify(fs.lstat)
-const readDir = promisify(fs.readdir)
-const mvFile = promisify(fs.rename)
+import * as fs from 'node:fs/promises'
+import * as path from 'node:path'
 
 class MigrationFilesManager {
 	constructor(public readonly directory: string) {}
 
 	public async createFile(content: string, name: string): Promise<string> {
 		const path = this.formatPath(name)
-		await fsWrite(path, content, { encoding: 'utf8' })
-		return await fsRealpath(path)
+		await fs.writeFile(path, content, { encoding: 'utf8' })
+		return await fs.realpath(path)
 	}
 
 	public async removeFile(name: string) {
 		const path = this.formatPath(name)
-		await fsRemove(path)
+		await fs.unlink(path)
 	}
 
 	public async moveFile(oldName: string, newName: string) {
-		await mvFile(this.formatPath(oldName), this.formatPath(newName))
+		await fs.rename(this.formatPath(oldName), this.formatPath(newName))
 	}
 
 	public async createDirIfNotExist(): Promise<void> {
 		try {
-			await mkdir(this.directory)
+			await fs.mkdir(this.directory)
 		} catch (e) {
 			if (!(e instanceof Error) || !('code' in e) || (e as any).code !== 'EEXIST') {
 				throw e
@@ -47,7 +37,7 @@ class MigrationFilesManager {
 			files
 				.filter(file => file.endsWith(`.json`))
 				.filter(async file => {
-					return (await lstatFile(`${this.directory}/${file}`)).isFile()
+					return (await fs.lstat(`${this.directory}/${file}`)).isFile()
 				}),
 		)
 		return filteredFiles.sort()
@@ -55,7 +45,7 @@ class MigrationFilesManager {
 
 	private async tryReadDir(): Promise<string[]> {
 		try {
-			return await readDir(this.directory)
+			return await fs.readdir(this.directory)
 		} catch (e) {
 			if (e instanceof Error && 'code' in e && (e as any).code === 'ENOENT') {
 				return []
@@ -72,7 +62,7 @@ class MigrationFilesManager {
 		const filesWithContent = files.map(async filename => ({
 			filename: filename,
 			path: `${this.directory}/${filename}`,
-			content: await readFile(`${this.directory}/${filename}`, { encoding: 'utf8' }),
+			content: await fs.readFile(`${this.directory}/${filename}`, { encoding: 'utf8' }),
 		}))
 
 		return await Promise.all(filesWithContent)

package/src/MigrationsResolver.ts

@@ -18,6 +18,7 @@ export class MigrationsResolver {
 				name: MigrationVersionHelper.extractName(filename),
 				formatVersion: parsed.formatVersion || VERSION_INITIAL,
 				modifications: parsed.modifications,
+				skippedErrors: parsed.skippedErrors ?? [],
 			}
 		})
 	}

package/src/SchemaDiffer.ts

@@ -44,14 +44,22 @@ import {
 import { ChangeViewNonViewDiffer, RemoveChangedFieldDiffer, RemoveChangedViewDiffer } from './modifications/differs'
 import { CreateIndexDiffer, RemoveIndexDiffer } from './modifications/indexes'
 import { SchemaWithMeta } from './modifications/utils/schemaMeta'
+import { UpdateSettingsDiffer } from './modifications/settings'
+
+type DiffOptions = { skipRecreateValidation?: boolean; skipInitialSchemaValidation?: boolean }
 
 export class SchemaDiffer {
 	constructor(private readonly schemaMigrator: SchemaMigrator) {}
 
-	diffSchemas(originalSchema: Schema, updatedSchema: Schema, checkRecreate: boolean = true): Migration.Modification[] {
-		const originalErrors = SchemaValidator.validate(originalSchema)
-		if (originalErrors.length > 0) {
-			throw new InvalidSchemaException('original schema is not valid', originalErrors)
+	diffSchemas(originalSchema: Schema, updatedSchema: Schema, {
+		skipInitialSchemaValidation = false,
+		skipRecreateValidation = false,
+	}: DiffOptions = {}): Migration.Modification[] {
+		if (!skipInitialSchemaValidation) {
+			const originalErrors = SchemaValidator.validate(originalSchema)
+			if (originalErrors.length > 0) {
+				throw new InvalidSchemaException('original schema is not valid', originalErrors)
+			}
 		}
 		const updatedErrors = SchemaValidator.validate(updatedSchema)
 		if (updatedErrors.length > 0) {
@@ -59,6 +67,7 @@ export class SchemaDiffer {
 		}
 
 		const differs: Differ[] = [
+			new UpdateSettingsDiffer(),
 			new ConvertOneToManyRelationDiffer(),
 			new ConvertOneHasManyToManyHasManyRelationDiffer(),
 			new RemoveUniqueConstraintDiffer(),
@@ -84,8 +93,8 @@ export class SchemaDiffer {
 			new RemoveChangedFieldDiffer(it => isRelation(it) && isInverseRelation(it)),
 			new CreateEntityDiffer(),
 			new CreateColumnDiffer(),
-			new CreateViewDiffer(),
 			new CreateRelationDiffer(),
+			new CreateViewDiffer(),
 			new CreateRelationInverseSideDiffer(),
 			new CreateUniqueConstraintDiffer(),
 			new CreateIndexDiffer(),
@@ -103,7 +112,7 @@ export class SchemaDiffer {
 		}
 
 
-		if (checkRecreate) {
+		if (!skipRecreateValidation) {
 			const { meta, ...appliedDiffsSchema2 } = appliedDiffsSchema as SchemaWithMeta
 			const errors = deepCompare(updatedSchema, appliedDiffsSchema2, [])
 			if (errors.length === 0) {

package/src/modifications/ModificationHandlerFactory.ts

@@ -30,6 +30,7 @@ import {
 import { patchValidationSchemaModification, updateValidationSchemaModification } from './validation'
 import { createIndexModification, removeIndexModification } from './indexes'
 import { SchemaWithMeta } from './utils/schemaMeta'
+import { updateSettingsModification } from './settings'
 
 
 class ModificationHandlerFactory {
@@ -47,6 +48,7 @@ namespace ModificationHandlerFactory {
 	type HandlerMap<D> = { [modificationName: string]: ModificationType<string, D> }
 
 	const handlers = [
+		updateSettingsModification,
 		updateAclSchemaModification,
 		patchAclSchemaModification,
 		createColumnModification,

package/src/modifications/settings/UpdateSettingsModification.ts

@@ -0,0 +1,87 @@
+import { MigrationBuilder } from '@contember/database-migrations'
+import { Schema, Settings } from '@contember/schema'
+import { SchemaUpdater } from '../utils/schemaUpdateUtils'
+import { createModificationType, Differ, ModificationHandler } from '../ModificationHandler'
+import deepEqual from 'fast-deep-equal'
+
+export class UpdateSettingsModificationHandler implements ModificationHandler<UpdateSettingsModificationData> {
+	constructor(
+		private readonly data: UpdateSettingsModificationData,
+	) {
+	}
+
+	public createSql(builder: MigrationBuilder): void {
+	}
+
+	public getSchemaUpdater(): SchemaUpdater {
+		return ({ schema }) => {
+			const { [this.data.key]: _, ...settings } = schema.settings
+			if (this.data.op === 'unset') {
+				return {
+					...schema,
+					settings,
+				}
+			}
+			return {
+				...schema,
+				settings: { ...settings, [this.data.key]: this.data.value },
+			}
+		}
+	}
+
+
+	describe() {
+		return { message: `Change settings of ${this.data.key}` }
+	}
+
+}
+
+export const updateSettingsModification = createModificationType({
+	id: 'updateSettings',
+	handler: UpdateSettingsModificationHandler,
+})
+
+export class UpdateSettingsDiffer implements Differ {
+	createDiff(originalSchema: Schema, updatedSchema: Schema) {
+		const allKeys = Array.from(new Set([
+			...Object.keys(originalSchema.settings),
+			...Object.keys(updatedSchema.settings),
+		] as (keyof Settings.Schema)[]))
+
+		return allKeys
+			.filter(key => !deepEqual(originalSchema.settings[key], updatedSchema.settings[key]))
+			.map(key => {
+				const value = updatedSchema.settings[key]
+				if (value === undefined) {
+					return updateSettingsModification.createModification({
+						op: 'unset',
+						key,
+					})
+				}
+				return updateSettingsModification.createModification({
+					op: 'set',
+					key,
+					value,
+				})
+			})
+	}
+}
+
+
+
+export interface SetSettingsModificationData<K extends keyof Settings.Schema = keyof Settings.Schema> {
+	op: 'set'
+	key: K
+	value: Settings.Schema[K]
+}
+
+
+export interface UnsetSettingsModificationData<K extends keyof Settings.Schema = keyof Settings.Schema> {
+	op: 'unset'
+	key: K
+}
+
+
+export type UpdateSettingsModificationData =
+	| SetSettingsModificationData
+	| UnsetSettingsModificationData

package/src/modifications/settings/index.ts

@@ -0,0 +1 @@
+export * from './UpdateSettingsModification'

package/src/modifications/utils/schemaUpdateUtils.ts

@@ -271,7 +271,14 @@ export const removeField = (entityName: string, fieldName: string, version: numb
 			updateModel(
 				updateEntity(entity.name, ({ entity }) => {
 					const { [field.name]: removed, ...fields } = entity.fields
-					return { ...entity, fields }
+					const indexes = Object.entries(entity.indexes).filter(([, index]) => !index.fields.includes(field.name))
+					const unique = Object.entries(entity.unique).filter(([, index]) => !index.fields.includes(field.name))
+					return {
+						...entity,
+						fields,
+						indexes: Object.fromEntries(indexes),
+						unique: Object.fromEntries(unique),
+					}
 				}),
 				isRelation(field) && isInverseRelation(field)
 					? updateEntity(

package/src/tsconfig.json

@@ -7,10 +7,10 @@
 	},
 	"references": [
 		{
-			"path": "../../database-migrations/src"
+			"path": "../../database/src"
 		},
 		{
-			"path": "../../engine-common/src"
+			"path": "../../database-migrations/src"
 		},
 		{
 			"path": "../../schema/src"

package/tests/cases/integration/createView.test.ts

@@ -86,3 +86,73 @@ testMigrations('create view', {
 	sql: SQL`CREATE VIEW "author_stats" AS SELECT 1;`,
 })
 
+
+namespace ViewAddRelationOriginalSchema {
+	export class Article {
+		title = def.stringColumn()
+	}
+
+
+	export class Category {
+		name = def.stringColumn()
+	}
+}
+
+namespace ViewAddRelationUpdateSchema {
+	export class Article {
+		title = def.stringColumn()
+		category = def.manyHasOne(Category)
+		stats = def.oneHasOneInverse(ArticleStats, 'article')
+	}
+
+	export class Category {
+		name = def.stringColumn()
+	}
+
+	@def.View('SELECT 1')
+	export class ArticleStats {
+		article = def.oneHasOne(Article, 'stats')
+		visitCount = def.intColumn()
+	}
+}
+
+
+testMigrations('create a relation and a view', {
+	originalSchema: def.createModel(ViewAddRelationOriginalSchema), updatedSchema: def.createModel(ViewAddRelationUpdateSchema), diff: [
+		{
+			modification: 'createRelation',
+			entityName: 'Article',
+			owningSide: { name: 'category', nullable: true, type: 'ManyHasOne', target: 'Category', joiningColumn: { columnName: 'category_id', onDelete: 'restrict' } },
+		},
+		{
+			modification: 'createView',
+			entity: { name: 'ArticleStats',
+				primary: 'id',
+				primaryColumn: 'id',
+				unique: {},
+				indexes: {},
+				fields: { id: { name: 'id', columnName: 'id', nullable: false, type: 'Uuid', columnType: 'uuid' },
+					article: { name: 'article', inversedBy: 'stats', nullable: true, type: 'OneHasOne', target: 'Article', joiningColumn: { columnName: 'article_id', onDelete: 'restrict' } },
+					visitCount: { name: 'visitCount', columnName: 'visit_count', nullable: true, type: 'Integer', columnType: 'integer' } },
+				tableName: 'article_stats',
+				eventLog: { enabled: true },
+				view: { sql: 'SELECT 1' } },
+		},
+		{
+			modification: 'createRelationInverseSide',
+			entityName: 'Article',
+			relation: {
+				name: 'stats',
+				ownedBy: 'article',
+				target: 'ArticleStats',
+				type: 'OneHasOne',
+				nullable: true,
+			},
+		},
+	],
+	sql: SQL`ALTER TABLE "article" ADD "category_id" uuid; 
+ALTER TABLE "article" ADD CONSTRAINT "fk_article_category_id_703b8b" FOREIGN KEY ("category_id") REFERENCES "category"("id") ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE; 
+CREATE INDEX "article_category_id_index" ON "article" ("category_id"); 
+CREATE VIEW "article_stats" AS SELECT 1;`,
+})
+

package/tests/cases/integration/removeField.test.ts

@@ -2,6 +2,8 @@ import { testMigrations } from '../../src/tests'
 import { SchemaBuilder } from '@contember/schema-definition'
 import { Model } from '@contember/schema'
 import { SQL } from '../../src/tags'
+import { SchemaDefinition as def } from '@contember/schema-definition'
+
 
 testMigrations('remove relation (many has one)', {
 	originalSchema: new SchemaBuilder()
@@ -128,3 +130,59 @@ testMigrations('remove relation inverse side', {
 	],
 	sql: SQL``,
 })
+
+
+namespace DropIndexOrigSchema {
+
+	@def.Unique('title', 'author')
+	@def.Index('title', 'author')
+	export class Article {
+		title = def.stringColumn()
+		author = def.manyHasOne(Author)
+	}
+
+	export class Author {
+		name = def.stringColumn()
+	}
+}
+
+namespace DropIndexUpSchema {
+	@def.Unique('title', 'author')
+	@def.Index('title', 'author')
+	export class Article {
+		title = def.stringColumn()
+		author = def.stringColumn()
+	}
+
+	export class Author {
+		name = def.stringColumn()
+	}
+}
+
+
+testMigrations('test drop index / unique when removing a field', {
+	originalSchema: def.createModel(DropIndexOrigSchema),
+	updatedSchema: def.createModel(DropIndexUpSchema),
+	diff: [{
+		modification: 'removeField',
+		entityName: 'Article',
+		fieldName: 'author',
+	}, {
+		modification: 'createColumn',
+		entityName: 'Article',
+		field: { name: 'author', columnName: 'author', nullable: true, type: 'String', columnType: 'text' },
+	}, {
+		modification: 'createUniqueConstraint',
+		entityName: 'Article',
+		unique: { name: 'unique_Article_title_author_7157ea', fields: ['title', 'author'] },
+	}, {
+		modification: 'createIndex',
+		entityName: 'Article',
+		index: { name: 'idx_Article_title_author_7157ea', fields: ['title', 'author'] },
+	}],
+	sql: SQL`ALTER TABLE "article" DROP "author_id";
+ALTER TABLE "article" ADD "author" text;
+ALTER TABLE "article" ADD CONSTRAINT "unique_Article_title_author_7157ea" UNIQUE ("title", "author");
+CREATE INDEX "idx_Article_title_author_7157ea" ON "article" ("title", "author");`,
+})
+

package/tests/scripts/checkCreateConsistentMigrations.ts

@@ -1,4 +1,4 @@
-import { relative } from 'path'
+import { relative } from 'node:path'
 import {
 	MigrationFilesManager,
 	MigrationsResolver,
@@ -18,7 +18,8 @@ import { emptySchema, schemaType } from '@contember/schema-utils'
 	for (const migration of await migrationsResolver.getMigrations()) {
 		const nextSchema = migrator.applyModifications(schema, migration.modifications, migration.formatVersion)
 		schemaType(nextSchema)
-		differ.diffSchemas(schema, nextSchema)
+		const { meta, ...nextSchemaWithoutMeta } = nextSchema
+		differ.diffSchemas(schema, nextSchemaWithoutMeta)
 
 		schema = nextSchema
 	}

package/tests/src/tests.ts

@@ -3,6 +3,7 @@ import { Acl, Model } from '@contember/schema'
 import { createMigrationBuilder } from '@contember/database-migrations'
 import { assert, describe, it } from 'vitest'
 import { SchemaWithMeta } from '../../src/modifications/utils/schemaMeta'
+import { emptySchema } from '@contember/schema-utils'
 
 const modificationFactory = new ModificationHandlerFactory(ModificationHandlerFactory.defaultFactoryMap)
 const schemaMigrator = new SchemaMigrator(modificationFactory)
@@ -28,17 +29,24 @@ export function testDiffSchemas(
 	updatedAcl: Acl.Schema = emptyAcl,
 ) {
 	const actualDiff = schemaDiffer.diffSchemas(
-		{ model: originalModel, acl: originalAcl, validation: {} },
-		{ model: updatedModel, acl: updatedAcl, validation: {} },
-		false,
+		{ ...emptySchema, model: originalModel, acl: originalAcl, validation: {} },
+		{ ...emptySchema, model: updatedModel, acl: updatedAcl, validation: {} },
+		{ skipRecreateValidation: true },
 	)
-	assert.deepStrictEqual(actualDiff, expectedDiff)
+	try {
+		assert.deepStrictEqual(actualDiff, expectedDiff)
+	} catch (e) {
+		// eslint-disable-next-line no-console
+		console.log(JSON.stringify(actualDiff))
+		throw e
+	}
 	const { meta, ...schema } = schemaMigrator.applyModifications(
-		{ model: originalModel, acl: originalAcl, validation: {} },
+		{ ...emptySchema, model: originalModel, acl: originalAcl, validation: {} },
 		actualDiff,
 		VERSION_LATEST,
 	) as SchemaWithMeta
 	assert.deepStrictEqual(schema, {
+		...emptySchema,
 		model: updatedModel,
 		acl: updatedAcl,
 		validation: {},
@@ -53,12 +61,13 @@ export function testApplyDiff(
 	expectedAcl: Acl.Schema = emptyAcl,
 ) {
 	const { meta, ...actualSchema } = schemaMigrator.applyModifications(
-		{ model: originalModel, acl: originalAcl, validation: {} },
+		{ ...emptySchema, model: originalModel, acl: originalAcl, validation: {} },
 		diff,
 		VERSION_LATEST,
 	) as SchemaWithMeta
 
 	assert.deepStrictEqual(actualSchema, {
+		...emptySchema,
 		model: expectedModel,
 		acl: expectedAcl,
 		validation: {},
@@ -66,7 +75,7 @@ export function testApplyDiff(
 }
 
 export function testGenerateSql(originalSchema: Model.Schema, diff: Migration.Modification[], expectedSql: string) {
-	let schema = { model: originalSchema, acl: emptyAcl, validation: {} }
+	let schema = { ...emptySchema, model: originalSchema, acl: emptyAcl, validation: {} }
 	const builder = createMigrationBuilder()
 	for (let { modification, ...data } of diff) {
 		const modificationHandler = modificationFactory.create(modification, data, schema, { formatVersion: VERSION_LATEST, systemSchema: 'system' })