@balena/pinejs 14.44.0-linear-runtime-migrator-e15bea04e85fb013aeed7d4770052b51747a619c → 15.0.0-delete-state-default-user-permissions-ba0732a0c5d0da9d1d5be818cb08cc898e86ebe3

package/src/config-loader/config-loader.ts

@@ -1,16 +1,9 @@
 import type * as Express from 'express';
 import type { AbstractSqlModel } from '@balena/abstract-sql-compiler';
 import type { Database } from '../database-layer/db';
+import type { Migration } from '../migrator/migrator';
 import type { AnyObject, Resolvable } from '../sbvr-api/common-types';
 
-import {
-	Migration,
-	Migrations,
-	defaultMigrationCategory,
-	MigrationCategories,
-	MigrationCategory,
-} from '../migrator/utils';
-
 import * as fs from 'fs';
 import * as _ from 'lodash';
 import * as path from 'path';
@@ -32,7 +25,9 @@ export interface Model {
 	modelText?: string;
 	abstractSql?: AbstractSqlModel;
 	migrationsPath?: string;
-	migrations?: Migrations;
+	migrations?: {
+		[index: string]: Migration;
+	};
 	initSqlPath?: string;
 	initSql?: string;
 	customServerCode?:
@@ -133,15 +128,31 @@ export const setup = (app: Express.Application) => {
 								},
 							);
 							if (user.permissions != null) {
-								await Promise.all(
+								const permissionIds = await Promise.all(
 									user.permissions.map(async (permissionName) => {
 										const permissionID = await permissionsCache[permissionName];
 										await getOrCreate(authApiTx, 'user__has__permission', {
 											user: userID,
 											permission: permissionID,
 										});
+										return permissionID;
 									}),
 								);
+								if (permissionIds.length > 0) {
+									await authApiTx.delete({
+										resource: 'user__has__permission',
+										options: {
+											$filter: {
+												user: userID,
+												$not: {
+													permission: {
+														$in: permissionIds,
+													},
+												},
+											},
+										},
+									});
+								}
 							}
 						} catch (e) {
 							e.message = `Could not create or find user "${user.username}": ${e.message}`;
@@ -256,62 +267,18 @@ export const setup = (app: Express.Application) => {
 						await Promise.all(
 							fileNames.map(async (filename) => {
 								const filePath = path.join(migrationsPath, filename);
-								const fileNameParts = filename.split('.', 3);
-								const fileExtension = path.extname(filename);
 								const [migrationKey] = filename.split('-', 1);
-								let migrationCategory: MigrationCategory =
-									defaultMigrationCategory;
-
-								if (fileNameParts.length === 3) {
-									if (fileNameParts[1] in MigrationCategories) {
-										migrationCategory = fileNameParts[1] as MigrationCategory;
-									} else {
-										console.error(
-											`Unrecognised migration file category ${
-												fileNameParts[1]
-											}, skipping: ${path.extname(filename)}`,
-										);
-										return;
-									}
-								}
 
-								/**
-								 *  helper to assign migrations with category level to model
-								 *  example migration file names:
-								 *
-								 *  key0-name.ts  				==> defaults startup migration
-								 *  key1-name1.sql 				==> defaults startup migration
-								 *  key2-name2.sync.sql 		==> explicit synchrony migration
-								 *  key3-name3.async.ts			==> async migration (async datafiller)
-								 *  key4-name4.async.sql		==> async migration (async datafiller)
-								 *
-								 */
-								const assignMigrationWithCategory = (
-									newMigrationKey: string,
-									newMigration: Migration,
-								) => {
-									const catMigrations = migrations[migrationCategory] || {};
-									if (typeof catMigrations === 'object') {
-										migrations[migrationCategory] = {
-											[newMigrationKey]: newMigration,
-											...catMigrations,
-										};
-									}
-								};
-
-								switch (fileExtension) {
+								switch (path.extname(filename)) {
 									case '.coffee':
 									case '.ts':
 									case '.js':
-										assignMigrationWithCategory(
-											migrationKey,
-											nodeRequire(filePath),
-										);
+										migrations[migrationKey] = nodeRequire(filePath);
 										break;
 									case '.sql':
-										assignMigrationWithCategory(
-											migrationKey,
-											await fs.promises.readFile(filePath, 'utf8'),
+										migrations[migrationKey] = await fs.promises.readFile(
+											filePath,
+											'utf8',
 										);
 										break;
 									default:
@@ -331,26 +298,12 @@ export const setup = (app: Express.Application) => {
 				}),
 			);
 			await loadConfig(configObj);
-			runAsyncMigrations(configObj); // async migrations will run in background - nothing to wait for
 		} catch (err) {
 			console.error('Error loading application config', err, err.stack);
 			process.exit(1);
 		}
 	};
 
-	const runAsyncMigrations = (data: Config): void => {
-		for (const model of data.models) {
-			if (
-				model.migrations != null &&
-				Object.keys(model.migrations).includes(MigrationCategories.async)
-			) {
-				sbvrUtils.executeAsyncModelMigration(
-					model as sbvrUtils.ExecutableModel,
-				);
-			}
-		}
-	};
-
 	return {
 		loadConfig,
 		loadApplicationConfig,

package/src/config-loader/env.ts

@@ -108,7 +108,4 @@ export const migrator = {
 	lockTimeout: 5 * 60 * 1000,
 	// Used to delay the failure on lock taking, to avoid spam taking
 	lockFailDelay: 20 * 1000,
-	asyncMigrationDefaultDelayMS: 1000,
-	asyncMigrationDefaultBackoffDelayMS: 60000,
-	asyncMigrationDefaultErrorThreshold: 10,
 };

package/src/migrator/migrations.sbvr

@@ -7,7 +7,6 @@ Term: executed migrations
 Term: lock time
 	Concept Type: Date Time (Type)
 
-
 Term: migration
 	Reference Scheme: model name
 	Database ID Field: model name
@@ -22,68 +21,3 @@ Term: migration lock
 
 Fact Type: migration lock has model name
 	Necessity: each migration lock has exactly one model name
-
-
-Term: migration key
-	Concept Type: Short Text (Type)
-Term: start time
-	Concept Type: Date Time (Type)
-Term: last run time
-	Concept Type: Date Time (Type)
-Term: run counter
-	Concept Type: Integer (Type)
-Term: migrated rows
-	Concept Type: Integer (Type)
-Term: error counter
-	Concept Type: Integer (Type)
-Term: error threshold
-	Concept Type: Integer (Type)
-Term: delayMS
-	Concept Type: Integer (Type)
-Term: backoffDelayMS
-	Concept Type: Integer (Type)
-Term: converged time 
-	Concept Type: Date Time (Type)
-Term: last error message
-	Concept Type: Text (Type)
-
-Term: migration status
-	Reference Scheme: migration key
-	Database ID Field: migration key
-
-Fact Type: migration status has migration key
-	Necessity: each migration status has exactly one migration key
-
-Fact Type: migration status has start time
-	Necessity: each migration status has at most one start time
-
-Fact Type: migration status has last run time
-	Necessity: each migration status has at most one last run time
-
-Fact Type: migration status has run counter
-	Necessity: each migration status has at most one run counter
-
-Fact Type: migration status has migrated rows
-	Necessity: each migration status has at most one migrated rows
-
-Fact Type: migration status has error counter
-	Necessity: each migration status has at most one error counter
-
-Fact Type: migration status has error threshold
-	Necessity: each migration status has at most one error threshold
-
-Fact Type: migration status has delayMS
-	Necessity: each migration status has at most one delayMS
-
-Fact Type: migration status has backoffDelayMS
-	Necessity: each migration status has at most one backoffDelayMS
-
-Fact Type: migration status is backoff
-
-Fact Type: migration status has converged time 
-	Necessity: each migration status has at most one converged time
-
-Fact Type: migration status has last error message
-	Necessity: each migration status has at most one last error message
-
-Fact Type: migration status should stop

package/src/migrator/migrator.ts

@@ -0,0 +1,278 @@
+import type { Tx } from '../database-layer/db';
+import type { Resolvable } from '../sbvr-api/common-types';
+import type { Config, Model } from '../config-loader/config-loader';
+
+import { Engines } from '@balena/abstract-sql-compiler';
+import * as _ from 'lodash';
+import { TypedError } from 'typed-error';
+import { migrator as migratorEnv } from '../config-loader/env';
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+import { delay } from '../sbvr-api/control-flow';
+
+// tslint:disable-next-line:no-var-requires
+const modelText: string = require('./migrations.sbvr');
+
+type ApiRootModel = Model & { apiRoot: string };
+
+type SbvrUtils = typeof sbvrUtils;
+
+type MigrationTuple = [string, Migration];
+
+export type MigrationFn = (tx: Tx, sbvrUtils: SbvrUtils) => Resolvable<void>;
+
+export type Migration = string | MigrationFn;
+
+export class MigrationError extends TypedError {}
+
+// Tagged template to convert binds from `?` format to the necessary output format,
+// eg `$1`/`$2`/etc for postgres
+const binds = (strings: TemplateStringsArray, ...bindNums: number[]) =>
+	strings
+		.map((str, i) => {
+			if (i === bindNums.length) {
+				return str;
+			}
+			if (i + 1 !== bindNums[i]) {
+				throw new SyntaxError('Migration sql binds must be sequential');
+			}
+			if (sbvrUtils.db.engine === Engines.postgres) {
+				return str + `$${bindNums[i]}`;
+			}
+			return str + `?`;
+		})
+		.join('');
+
+export const postRun = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { initSql } = model;
+	if (initSql == null) {
+		return;
+	}
+
+	const modelName = model.apiRoot;
+
+	const exists = await checkModelAlreadyExists(tx, modelName);
+	if (!exists) {
+		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+			'First time executing, running init script',
+		);
+		await lockMigrations(tx, modelName, async () => {
+			await tx.executeSql(initSql);
+		});
+	}
+};
+
+export const run = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { migrations } = model;
+	if (migrations == null || _.isEmpty(migrations)) {
+		return;
+	}
+
+	const modelName = model.apiRoot;
+
+	// migrations only run if the model has been executed before,
+	// to make changes that can't be automatically applied
+	const exists = await checkModelAlreadyExists(tx, modelName);
+	if (!exists) {
+		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+			'First time model has executed, skipping migrations',
+		);
+
+		return await setExecutedMigrations(tx, modelName, Object.keys(migrations));
+	}
+	await lockMigrations(tx, modelName, async () => {
+		const executedMigrations = await getExecutedMigrations(tx, modelName);
+		const pendingMigrations = filterAndSortPendingMigrations(
+			migrations,
+			executedMigrations,
+		);
+		if (pendingMigrations.length === 0) {
+			return;
+		}
+
+		const newlyExecutedMigrations = await executeMigrations(
+			tx,
+			pendingMigrations,
+		);
+		await setExecutedMigrations(tx, modelName, [
+			...executedMigrations,
+			...newlyExecutedMigrations,
+		]);
+	});
+};
+
+const checkModelAlreadyExists = async (
+	tx: Tx,
+	modelName: string,
+): Promise<boolean> => {
+	const result = await tx.tableList("name = 'migration'");
+	if (result.rows.length === 0) {
+		return false;
+	}
+	const { rows } = await tx.executeSql(
+		binds`
+SELECT 1
+FROM "model"
+WHERE "model"."is of-vocabulary" = ${1}
+LIMIT 1`,
+		[modelName],
+	);
+
+	return rows.length > 0;
+};
+
+const getExecutedMigrations = async (
+	tx: Tx,
+	modelName: string,
+): Promise<string[]> => {
+	const { rows } = await tx.executeSql(
+		binds`
+SELECT "migration"."executed migrations" AS "executed_migrations"
+FROM "migration"
+WHERE "migration"."model name" = ${1}`,
+		[modelName],
+	);
+
+	const data = rows[0];
+	if (data == null) {
+		return [];
+	}
+
+	return JSON.parse(data.executed_migrations) as string[];
+};
+
+const setExecutedMigrations = async (
+	tx: Tx,
+	modelName: string,
+	executedMigrations: string[],
+): Promise<void> => {
+	const stringifiedMigrations = JSON.stringify(executedMigrations);
+
+	const result = await tx.tableList("name = 'migration'");
+	if (result.rows.length === 0) {
+		return;
+	}
+
+	const { rowsAffected } = await tx.executeSql(
+		binds`
+UPDATE "migration"
+SET "model name" = ${1},
+"executed migrations" = ${2}
+WHERE "migration"."model name" = ${3}`,
+		[modelName, stringifiedMigrations, modelName],
+	);
+
+	if (rowsAffected === 0) {
+		await tx.executeSql(
+			binds`
+INSERT INTO "migration" ("model name", "executed migrations")
+VALUES (${1}, ${2})`,
+			[modelName, stringifiedMigrations],
+		);
+	}
+};
+
+// turns {"key1": migration, "key3": migration, "key2": migration}
+// into  [["key1", migration], ["key2", migration], ["key3", migration]]
+const filterAndSortPendingMigrations = (
+	migrations: NonNullable<Model['migrations']>,
+	executedMigrations: string[],
+): MigrationTuple[] =>
+	(_(migrations).omit(executedMigrations) as _.Object<typeof migrations>)
+		.toPairs()
+		.sortBy(([migrationKey]) => migrationKey)
+		.value();
+
+const lockMigrations = async <T>(
+	tx: Tx,
+	modelName: string,
+	fn: () => Promise<T>,
+): Promise<T> => {
+	try {
+		await tx.executeSql(
+			binds`
+DELETE FROM "migration lock"
+WHERE "model name" = ${1}
+AND "created at" < ${2}`,
+			[modelName, new Date(Date.now() - migratorEnv.lockTimeout)],
+		);
+		await tx.executeSql(
+			binds`
+INSERT INTO "migration lock" ("model name")
+VALUES (${1})`,
+			[modelName],
+		);
+	} catch (err) {
+		await delay(migratorEnv.lockFailDelay);
+		throw err;
+	}
+	try {
+		return await fn();
+	} finally {
+		try {
+			await tx.executeSql(
+				binds`
+DELETE FROM "migration lock"
+WHERE "model name" = ${1}`,
+				[modelName],
+			);
+		} catch {
+			// We ignore errors here as it's mostly likely caused by the migration failing and
+			// rolling back the transaction, and if we rethrow here we'll overwrite the real error
+			// making it much harder for users to see what went wrong and fix it
+		}
+	}
+};
+
+const executeMigrations = async (
+	tx: Tx,
+	migrations: MigrationTuple[] = [],
+): Promise<string[]> => {
+	try {
+		for (const migration of migrations) {
+			await executeMigration(tx, migration);
+		}
+	} catch (err) {
+		(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+			'Error while executing migrations, rolled back',
+		);
+		throw new MigrationError(err);
+	}
+	return migrations.map(([migrationKey]) => migrationKey); // return migration keys
+};
+
+const executeMigration = async (
+	tx: Tx,
+	[key, migration]: MigrationTuple,
+): Promise<void> => {
+	(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+		`Running migration ${JSON.stringify(key)}`,
+	);
+
+	if (typeof migration === 'function') {
+		await migration(tx, sbvrUtils);
+	} else if (typeof migration === 'string') {
+		await tx.executeSql(migration);
+	} else {
+		throw new MigrationError(`Invalid migration type: ${typeof migration}`);
+	}
+};
+
+export const config: Config = {
+	models: [
+		{
+			modelName: 'migrations',
+			apiRoot: 'migrations',
+			modelText,
+			migrations: {
+				'11.0.0-modified-at': `
+					ALTER TABLE "migration"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'11.0.1-modified-at': `
+					ALTER TABLE "migration lock"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+			},
+		},
+	],
+};

package/src/sbvr-api/sbvr-utils.ts

@@ -33,8 +33,7 @@ import { PinejsClientCore, PromiseResultTypes } from 'pinejs-client-core';
 
 import { ExtendedSBVRParser } from '../extended-sbvr-parser/extended-sbvr-parser';
 
-import * as asyncMigrator from '../migrator/async';
-import * as syncMigrator from '../migrator/sync';
+import * as migrator from '../migrator/migrator';
 import { generateODataMetadata } from '../odata-metadata/odata-metadata-generator';
 
 // tslint:disable-next-line:no-var-requires
@@ -431,20 +430,6 @@ export const generateModels = (
 	return { vocab, se, lf, abstractSql, sql, odataMetadata };
 };
 
-export const executeAsyncModelMigration = (
-	model: ExecutableModel,
-): Promise<void> => executeAsyncModelMigrations([model]);
-
-export const executeAsyncModelMigrations = async (
-	execModels: ExecutableModel[],
-): Promise<void> => {
-	await Promise.all(
-		execModels.map(async (model) => {
-			await asyncMigrator.run(model);
-		}),
-	);
-};
-
 export const executeModel = (
 	tx: Db.Tx,
 	model: ExecutableModel,
@@ -459,7 +444,7 @@ export const executeModels = async (
 			execModels.map(async (model) => {
 				const { apiRoot } = model;
 
-				await syncMigrator.run(tx, model);
+				await migrator.run(tx, model);
 				const compiledModel = generateModels(model, db.engine);
 
 				// Create tables related to terms and fact types
@@ -476,7 +461,7 @@ export const executeModels = async (
 					}
 					await promise;
 				}
-				await syncMigrator.postRun(tx, model);
+				await migrator.postRun(tx, model);
 
 				odataResponse.prepareModel(compiledModel.abstractSql);
 				deepFreeze(compiledModel.abstractSql);

package/src/server-glue/module.ts

@@ -4,8 +4,7 @@ import './sbvr-loader';
 
 import * as dbModule from '../database-layer/db';
 import * as configLoader from '../config-loader/config-loader';
-import * as migrator from '../migrator/sync';
-import * as migratorUtils from '../migrator/utils';
+import * as migrator from '../migrator/migrator';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils';
 
@@ -18,7 +17,7 @@ export * as env from '../config-loader/env';
 export * as types from '../sbvr-api/common-types';
 export * as hooks from '../sbvr-api/hooks';
 export type { configLoader as ConfigLoader };
-export type { migratorUtils as Migrator };
+export type { migrator as Migrator };
 
 let envDatabaseOptions: dbModule.DatabaseOptions<string>;
 if (dbModule.engines.websql != null) {

package/out/migrator/async.d.ts

@@ -1,6 +0,0 @@
-import type { Model } from '../config-loader/config-loader';
-declare type ApiRootModel = Model & {
-    apiRoot: string;
-};
-export declare const run: (model: ApiRootModel) => Promise<void>;
-export {};