@balena/pinejs 15.0.0-delete-state-default-user-permissions-ba0732a0c5d0da9d1d5be818cb08cc898e86ebe3 → 15.0.0-delete-state-default-user-permissions-3639cb4f82f3d7c9636142a31f4df182080cef3f

package/src/config-loader/config-loader.ts

@@ -1,9 +1,15 @@
 import type * as Express from 'express';
 import type { AbstractSqlModel } from '@balena/abstract-sql-compiler';
 import type { Database } from '../database-layer/db';
-import type { Migration } from '../migrator/migrator';
 import type { AnyObject, Resolvable } from '../sbvr-api/common-types';
 
+import {
+	Migration,
+	Migrations,
+	defaultMigrationCategory,
+	MigrationCategories,
+} from '../migrator/utils';
+
 import * as fs from 'fs';
 import * as _ from 'lodash';
 import * as path from 'path';
@@ -25,9 +31,7 @@ export interface Model {
 	modelText?: string;
 	abstractSql?: AbstractSqlModel;
 	migrationsPath?: string;
-	migrations?: {
-		[index: string]: Migration;
-	};
+	migrations?: Migrations;
 	initSqlPath?: string;
 	initSql?: string;
 	customServerCode?:
@@ -138,21 +142,22 @@ export const setup = (app: Express.Application) => {
 										return permissionID;
 									}),
 								);
-								if (permissionIds.length > 0) {
-									await authApiTx.delete({
-										resource: 'user__has__permission',
-										options: {
-											$filter: {
-												user: userID,
+
+								await authApiTx.delete({
+									resource: 'user__has__permission',
+									options: {
+										$filter: {
+											user: userID,
+											...(permissionIds.length > 0 && {
 												$not: {
 													permission: {
 														$in: permissionIds,
 													},
 												},
-											},
+											}),
 										},
-									});
-								}
+									},
+								});
 							}
 						} catch (e) {
 							e.message = `Could not create or find user "${user.username}": ${e.message}`;
@@ -267,18 +272,59 @@ export const setup = (app: Express.Application) => {
 						await Promise.all(
 							fileNames.map(async (filename) => {
 								const filePath = path.join(migrationsPath, filename);
+								const fileNameParts = filename.split('.', 3);
+								const fileExtension = path.extname(filename);
 								const [migrationKey] = filename.split('-', 1);
+								let migrationCategory = defaultMigrationCategory;
 
-								switch (path.extname(filename)) {
+								if (fileNameParts.length === 3) {
+									if (fileNameParts[1] in MigrationCategories) {
+										migrationCategory = fileNameParts[1] as MigrationCategories;
+									} else {
+										console.error(
+											`Unrecognised migration file category ${
+												fileNameParts[1]
+											}, skipping: ${path.extname(filename)}`,
+										);
+										return;
+									}
+								}
+
+								/**
+								 *  helper to assign migrations with category level to model
+								 *  example migration file names:
+								 *
+								 *  key0-name.ts  				==> defaults startup migration
+								 *  key1-name1.sql 				==> defaults startup migration
+								 *  key2-name2.sync.sql 		==> explicit synchrony migration
+								 *
+								 */
+								const assignMigrationWithCategory = (
+									newMigrationKey: string,
+									newMigration: Migration,
+								) => {
+									const catMigrations = migrations[migrationCategory] || {};
+									if (typeof catMigrations === 'object') {
+										migrations[migrationCategory] = {
+											[newMigrationKey]: newMigration,
+											...catMigrations,
+										};
+									}
+								};
+
+								switch (fileExtension) {
 									case '.coffee':
 									case '.ts':
 									case '.js':
-										migrations[migrationKey] = nodeRequire(filePath);
+										assignMigrationWithCategory(
+											migrationKey,
+											nodeRequire(filePath),
+										);
 										break;
 									case '.sql':
-										migrations[migrationKey] = await fs.promises.readFile(
-											filePath,
-											'utf8',
+										assignMigrationWithCategory(
+											migrationKey,
+											await fs.promises.readFile(filePath, 'utf8'),
 										);
 										break;
 									default:

package/src/config-loader/env.ts

@@ -58,7 +58,9 @@ import memoizeWeak = require('memoizee/weak');
 export const createCache = <T extends (...args: any[]) => any>(
 	cacheName: keyof typeof cache,
 	fn: T,
-	opts?: CacheFnOpts<T>,
+	// TODO: Mark this as optional once TS is able to infer the `normalizer` types
+	// when the `weak` differentiating property is not provided.
+	opts: CacheFnOpts<T>,
 ) => {
 	const cacheOpts = cache[cacheName];
 	if (cacheOpts === false) {

package/src/migrator/sync.ts

@@ -0,0 +1,171 @@
+import {
+	modelText,
+	MigrationTuple,
+	MigrationError,
+	defaultMigrationCategory,
+	checkModelAlreadyExists,
+	setExecutedMigrations,
+	getExecutedMigrations,
+	lockMigrations,
+	RunnableMigrations,
+} from './utils';
+import type { Tx } from '../database-layer/db';
+import type { Config, Model } from '../config-loader/config-loader';
+
+import * as _ from 'lodash';
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+
+type ApiRootModel = Model & { apiRoot: string };
+
+export const postRun = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { initSql } = model;
+	if (initSql == null) {
+		return;
+	}
+
+	const modelName = model.apiRoot;
+
+	const exists = await checkModelAlreadyExists(tx, modelName);
+	if (!exists) {
+		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+			'First time executing, running init script',
+		);
+
+		await lockMigrations(tx, modelName, async () => {
+			try {
+				await tx.executeSql(initSql);
+			} catch (err) {
+				(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+					`initSql execution error ${err} `,
+				);
+				throw new MigrationError(err);
+			}
+		});
+	}
+};
+
+export const run = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { migrations } = model;
+	if (migrations == null || _.isEmpty(migrations)) {
+		return;
+	}
+	const defaultMigrations = migrations[defaultMigrationCategory];
+	const runMigrations: RunnableMigrations =
+		defaultMigrationCategory in migrations
+			? typeof defaultMigrations === 'object'
+				? defaultMigrations
+				: {}
+			: migrations;
+
+	return $run(tx, model, runMigrations);
+};
+
+const $run = async (
+	tx: Tx,
+	model: ApiRootModel,
+	migrations: RunnableMigrations,
+): Promise<void> => {
+	const modelName = model.apiRoot;
+
+	// migrations only run if the model has been executed before,
+	// to make changes that can't be automatically applied
+	const exists = await checkModelAlreadyExists(tx, modelName);
+	if (!exists) {
+		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+			'First time model has executed, skipping migrations',
+		);
+
+		return await setExecutedMigrations(tx, modelName, Object.keys(migrations));
+	}
+	await lockMigrations(tx, modelName, async () => {
+		try {
+			const executedMigrations = await getExecutedMigrations(tx, modelName);
+			const pendingMigrations = filterAndSortPendingMigrations(
+				migrations,
+				executedMigrations,
+			);
+			if (pendingMigrations.length === 0) {
+				return;
+			}
+
+			const newlyExecutedMigrations = await executeMigrations(
+				tx,
+				pendingMigrations,
+			);
+			await setExecutedMigrations(tx, modelName, [
+				...executedMigrations,
+				...newlyExecutedMigrations,
+			]);
+		} catch (err) {
+			(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+				`Failed to executed synchronous migrations from api root model ${err}`,
+			);
+			throw new MigrationError(err);
+		}
+	});
+};
+
+// turns {"key1": migration, "key3": migration, "key2": migration}
+// into  [["key1", migration], ["key2", migration], ["key3", migration]]
+const filterAndSortPendingMigrations = (
+	migrations: NonNullable<RunnableMigrations>,
+	executedMigrations: string[],
+): MigrationTuple[] =>
+	(_(migrations).omit(executedMigrations) as _.Object<typeof migrations>)
+		.toPairs()
+		.sortBy(([migrationKey]) => migrationKey)
+		.value();
+
+const executeMigrations = async (
+	tx: Tx,
+	migrations: MigrationTuple[] = [],
+): Promise<string[]> => {
+	try {
+		for (const migration of migrations) {
+			await executeMigration(tx, migration);
+		}
+	} catch (err) {
+		(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+			'Error while executing migrations, rolled back',
+		);
+		throw new MigrationError(err);
+	}
+	return migrations.map(([migrationKey]) => migrationKey); // return migration keys
+};
+
+const executeMigration = async (
+	tx: Tx,
+	[key, migration]: MigrationTuple,
+): Promise<void> => {
+	(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+		`Running migration ${JSON.stringify(key)}`,
+	);
+
+	if (typeof migration === 'function') {
+		await migration(tx, sbvrUtils);
+	} else if (typeof migration === 'string') {
+		await tx.executeSql(migration);
+	} else {
+		throw new MigrationError(`Invalid migration type: ${typeof migration}`);
+	}
+};
+
+export const config: Config = {
+	models: [
+		{
+			modelName: 'migrations',
+			apiRoot: 'migrations',
+			modelText,
+			migrations: {
+				'11.0.0-modified-at': `
+					ALTER TABLE "migration"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'11.0.1-modified-at': `
+					ALTER TABLE "migration lock"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+			},
+		},
+	],
+};

package/src/migrator/utils.ts

@@ -0,0 +1,160 @@
+import type { Tx } from '../database-layer/db';
+import type { Resolvable } from '../sbvr-api/common-types';
+
+import { Engines } from '@balena/abstract-sql-compiler';
+import * as _ from 'lodash';
+import { TypedError } from 'typed-error';
+import { migrator as migratorEnv } from '../config-loader/env';
+export { migrator as migratorEnv } from '../config-loader/env';
+import { delay } from '../sbvr-api/control-flow';
+
+// tslint:disable-next-line:no-var-requires
+export const modelText = require('./migrations.sbvr');
+
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+export enum MigrationCategories {
+	'sync' = 'sync',
+}
+export const defaultMigrationCategory = MigrationCategories.sync;
+export type CategorizedMigrations = {
+	[key in MigrationCategories]: RunnableMigrations;
+};
+
+type SbvrUtils = typeof sbvrUtils;
+export type MigrationTuple = [string, Migration];
+export type MigrationFn = (tx: Tx, sbvrUtils: SbvrUtils) => Resolvable<void>;
+export type Migration = string | MigrationFn;
+export type RunnableMigrations = { [key: string]: Migration };
+export type Migrations = CategorizedMigrations | RunnableMigrations;
+
+export class MigrationError extends TypedError {}
+
+// Tagged template to convert binds from `?` format to the necessary output format,
+// eg `$1`/`$2`/etc for postgres
+export const binds = (strings: TemplateStringsArray, ...bindNums: number[]) =>
+	strings
+		.map((str, i) => {
+			if (i === bindNums.length) {
+				return str;
+			}
+			if (i + 1 !== bindNums[i]) {
+				throw new SyntaxError('Migration sql binds must be sequential');
+			}
+			if (sbvrUtils.db.engine === Engines.postgres) {
+				return str + `$${bindNums[i]}`;
+			}
+			return str + `?`;
+		})
+		.join('');
+
+export const lockMigrations = async <T>(
+	tx: Tx,
+	modelName: string,
+	fn: () => Promise<T>,
+): Promise<T> => {
+	try {
+		await tx.executeSql(
+			binds`
+DELETE FROM "migration lock"
+WHERE "model name" = ${1}
+AND "created at" < ${2}`,
+			[modelName, new Date(Date.now() - migratorEnv.lockTimeout)],
+		);
+		await tx.executeSql(
+			binds`
+INSERT INTO "migration lock" ("model name")
+VALUES (${1})`,
+			[modelName],
+		);
+	} catch (err) {
+		await delay(migratorEnv.lockFailDelay);
+		throw err;
+	}
+	try {
+		return await fn();
+	} finally {
+		try {
+			await tx.executeSql(
+				binds`
+DELETE FROM "migration lock"
+WHERE "model name" = ${1}`,
+				[modelName],
+			);
+		} catch {
+			// We ignore errors here as it's mostly likely caused by the migration failing and
+			// rolling back the transaction, and if we rethrow here we'll overwrite the real error
+			// making it much harder for users to see what went wrong and fix it
+		}
+	}
+};
+
+export const checkModelAlreadyExists = async (
+	tx: Tx,
+	modelName: string,
+): Promise<boolean> => {
+	const result = await tx.tableList("name = 'migration'");
+	if (result.rows.length === 0) {
+		return false;
+	}
+	const { rows } = await tx.executeSql(
+		binds`
+SELECT 1
+FROM "model"
+WHERE "model"."is of-vocabulary" = ${1}
+LIMIT 1`,
+		[modelName],
+	);
+
+	return rows.length > 0;
+};
+
+export const setExecutedMigrations = async (
+	tx: Tx,
+	modelName: string,
+	executedMigrations: string[],
+): Promise<void> => {
+	const stringifiedMigrations = JSON.stringify(executedMigrations);
+
+	const result = await tx.tableList("name = 'migration'");
+	if (result.rows.length === 0) {
+		return;
+	}
+
+	const { rowsAffected } = await tx.executeSql(
+		binds`
+UPDATE "migration"
+SET "model name" = ${1},
+"executed migrations" = ${2}
+WHERE "migration"."model name" = ${3}`,
+		[modelName, stringifiedMigrations, modelName],
+	);
+
+	if (rowsAffected === 0) {
+		await tx.executeSql(
+			binds`
+INSERT INTO "migration" ("model name", "executed migrations")
+VALUES (${1}, ${2})`,
+			[modelName, stringifiedMigrations],
+		);
+	}
+};
+
+export const getExecutedMigrations = async (
+	tx: Tx,
+	modelName: string,
+): Promise<string[]> => {
+	const { rows } = await tx.executeSql(
+		binds`
+SELECT "migration"."executed migrations" AS "executed_migrations"
+FROM "migration"
+WHERE "migration"."model name" = ${1}`,
+		[modelName],
+	);
+
+	const data = rows[0];
+	if (data == null) {
+		return [];
+	}
+
+	return JSON.parse(data.executed_migrations) as string[];
+};

package/src/sbvr-api/permissions.ts

@@ -20,6 +20,7 @@ import type {
 	ODataQuery,
 	SupportedMethod,
 } from '@balena/odata-parser';
+import type { Tx } from '../database-layer/db';
 import type { ApiKey, User } from '../sbvr-api/sbvr-utils';
 import type { AnyObject } from './common-types';
 
@@ -345,7 +346,6 @@ const getPermissionsLookup = env.createCache(
 		return permissionsLookup;
 	},
 	{
-		weak: undefined,
 		normalizer: ([permissions, guestPermissions]) =>
 			// When guestPermissions is present it should always be the same, so we can key by presence not content
 			`${permissions}${guestPermissions == null}`,
@@ -1213,19 +1213,27 @@ const $getUserPermissions = (() => {
 	);
 	return env.createCache(
 		'userPermissions',
-		async (userId: number) => {
-			const permissions = (await getUserPermissionsQuery()({
-				userId,
-			})) as Array<{ name: string }>;
+		async (userId: number, tx?: Tx) => {
+			const permissions = (await getUserPermissionsQuery()(
+				{
+					userId,
+				},
+				undefined,
+				{ tx },
+			)) as Array<{ name: string }>;
 			return permissions.map((permission) => permission.name);
 		},
 		{
 			primitive: true,
 			promise: true,
+			normalizer: ([userId]) => `${userId}`,
 		},
 	);
 })();
-export const getUserPermissions = async (userId: number): Promise<string[]> => {
+export const getUserPermissions = async (
+	userId: number,
+	tx?: Tx,
+): Promise<string[]> => {
 	if (typeof userId === 'string') {
 		userId = parseInt(userId, 10);
 	}
@@ -1233,7 +1241,7 @@ export const getUserPermissions = async (userId: number): Promise<string[]> => {
 		throw new Error(`User ID has to be numeric, got: ${typeof userId}`);
 	}
 	try {
-		return await $getUserPermissions(userId);
+		return await $getUserPermissions(userId, tx);
 	} catch (err) {
 		sbvrUtils.api.Auth.logger.error('Error loading user permissions', err);
 		throw err;
@@ -1336,26 +1344,32 @@ const $getApiKeyPermissions = (() => {
 	);
 	return env.createCache(
 		'apiKeyPermissions',
-		async (apiKey: string) => {
-			const permissions = (await getApiKeyPermissionsQuery()({
-				apiKey,
-			})) as Array<{ name: string }>;
+		async (apiKey: string, tx?: Tx) => {
+			const permissions = (await getApiKeyPermissionsQuery()(
+				{
+					apiKey,
+				},
+				undefined,
+				{ tx },
+			)) as Array<{ name: string }>;
 			return permissions.map((permission) => permission.name);
 		},
 		{
 			primitive: true,
 			promise: true,
+			normalizer: ([apiKey]) => apiKey,
 		},
 	);
 })();
 export const getApiKeyPermissions = async (
 	apiKey: string,
+	tx?: Tx,
 ): Promise<string[]> => {
 	if (typeof apiKey !== 'string') {
 		throw new Error('API key has to be a string, got: ' + typeof apiKey);
 	}
 	try {
-		return await $getApiKeyPermissions(apiKey);
+		return await $getApiKeyPermissions(apiKey, tx);
 	} catch (err) {
 		sbvrUtils.api.Auth.logger.error('Error loading api key permissions', err);
 		throw err;
@@ -1386,10 +1400,14 @@ const getApiKeyActorId = (() => {
 	const apiActorPermissionError = new PermissionError();
 	return env.createCache(
 		'apiKeyActorId',
-		async (apiKey: string) => {
-			const apiKeyResult = await getApiKeyActorIdQuery()({
-				apiKey,
-			});
+		async (apiKey: string, tx?: Tx) => {
+			const apiKeyResult = await getApiKeyActorIdQuery()(
+				{
+					apiKey,
+				},
+				undefined,
+				{ tx },
+			);
 			if (apiKeyResult == null) {
 				// We reuse a constant permission error here as it will be cached, and
 				// using a single error instance can drastically reduce the memory used
@@ -1404,6 +1422,7 @@ const getApiKeyActorId = (() => {
 		{
 			promise: true,
 			primitive: true,
+			normalizer: ([apiKey]) => apiKey,
 		},
 	);
 })();
@@ -1411,13 +1430,14 @@ const getApiKeyActorId = (() => {
 const checkApiKey = async (
 	req: PermissionReq,
 	apiKey: string,
+	tx?: Tx,
 ): Promise<PermissionReq['apiKey']> => {
 	if (apiKey == null || req.apiKey != null) {
 		return;
 	}
 	let permissions: string[];
 	try {
-		permissions = await getApiKeyPermissions(apiKey);
+		permissions = await getApiKeyPermissions(apiKey, tx);
 	} catch (err) {
 		console.warn('Error with API key:', err);
 		// Ignore errors getting the api key and just use an empty permissions object.
@@ -1425,7 +1445,7 @@ const checkApiKey = async (
 	}
 	let actor;
 	if (permissions.length > 0) {
-		actor = await getApiKeyActorId(apiKey);
+		actor = await getApiKeyActorId(apiKey, tx);
 	}
 	const resolvedApiKey: PermissionReq['apiKey'] = {
 		key: apiKey,
@@ -1440,6 +1460,8 @@ const checkApiKey = async (
 export const resolveAuthHeader = async (
 	req: Express.Request,
 	expectedScheme = 'Bearer',
+	// TODO: Consider making tx the second argument in the next major
+	tx?: Tx,
 ): Promise<PermissionReq['apiKey']> => {
 	const auth = req.header('Authorization');
 	if (!auth) {
@@ -1456,7 +1478,7 @@ export const resolveAuthHeader = async (
 		return;
 	}
 
-	return await checkApiKey(req, apiKey);
+	return await checkApiKey(req, apiKey, tx);
 };
 
 export const customAuthorizationMiddleware = (expectedScheme = 'Bearer') => {
@@ -1483,10 +1505,12 @@ export const authorizationMiddleware = customAuthorizationMiddleware();
 export const resolveApiKey = async (
 	req: HookReq | Express.Request,
 	paramName = 'apikey',
+	// TODO: Consider making tx the second argument in the next major
+	tx?: Tx,
 ): Promise<PermissionReq['apiKey']> => {
 	const apiKey =
 		req.params[paramName] ?? req.body[paramName] ?? req.query[paramName];
-	return await checkApiKey(req, apiKey);
+	return await checkApiKey(req, apiKey, tx);
 };
 
 export const customApiKeyMiddleware = (paramName = 'apikey') => {

package/src/sbvr-api/sbvr-utils.ts

@@ -33,7 +33,7 @@ import { PinejsClientCore, PromiseResultTypes } from 'pinejs-client-core';
 
 import { ExtendedSBVRParser } from '../extended-sbvr-parser/extended-sbvr-parser';
 
-import * as migrator from '../migrator/migrator';
+import * as syncMigrator from '../migrator/sync';
 import { generateODataMetadata } from '../odata-metadata/odata-metadata-generator';
 
 // tslint:disable-next-line:no-var-requires
@@ -444,7 +444,7 @@ export const executeModels = async (
 			execModels.map(async (model) => {
 				const { apiRoot } = model;
 
-				await migrator.run(tx, model);
+				await syncMigrator.run(tx, model);
 				const compiledModel = generateModels(model, db.engine);
 
 				// Create tables related to terms and fact types
@@ -461,7 +461,7 @@ export const executeModels = async (
 					}
 					await promise;
 				}
-				await migrator.postRun(tx, model);
+				await syncMigrator.postRun(tx, model);
 
 				odataResponse.prepareModel(compiledModel.abstractSql);
 				deepFreeze(compiledModel.abstractSql);

package/src/server-glue/module.ts

@@ -4,7 +4,8 @@ import './sbvr-loader';
 
 import * as dbModule from '../database-layer/db';
 import * as configLoader from '../config-loader/config-loader';
-import * as migrator from '../migrator/migrator';
+import * as migrator from '../migrator/sync';
+import * as migratorUtils from '../migrator/utils';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils';
 
@@ -17,7 +18,7 @@ export * as env from '../config-loader/env';
 export * as types from '../sbvr-api/common-types';
 export * as hooks from '../sbvr-api/hooks';
 export type { configLoader as ConfigLoader };
-export type { migrator as Migrator };
+export type { migratorUtils as Migrator };
 
 let envDatabaseOptions: dbModule.DatabaseOptions<string>;
 if (dbModule.engines.websql != null) {